diff --git a/src/code/issue4/.gitignore b/src/code/issue4/.gitignore new file mode 100644 index 0000000..008ea26 --- /dev/null +++ b/src/code/issue4/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.py[cod] +reports/ +issue4-reproducibility-report.json diff --git a/src/code/issue4/README-2026.md b/src/code/issue4/README-2026.md new file mode 100644 index 0000000..2952432 --- /dev/null +++ b/src/code/issue4/README-2026.md @@ -0,0 +1,75 @@ +# Issue 4:集合通信 Bitwise 可复现性诊断工具 + +`reproducibility_diagnoser.py` 对完全相同的输入重复执行 AllReduce 或 +Reduce-Scatter,并按 `run / call / rank` 保存逐位摘要。发现差异时,报告会给出: + +- 首次出现差异的 collective 调用点和 rank; +- 首个差异元素、差异元素总数; +- 最大绝对误差和最大相对误差; +- 后续差异随 run/call 的演化; +- PyTorch/CUDA/NCCL 版本、GPU 型号和影响 NCCL 选择的环境变量。 + +## 单卡冒烟验证 + +NCCL 只在 Linux 版 PyTorch 中提供,Windows 版不含 NCCL。在 Windows 上开发时请在带 +CUDA 透传的 WSL2 里运行。以下命令在仓库根目录执行: + +```bash +# 无依赖(不需要 GPU):演示 FP32 归约顺序为何会改变结果位模式 +python src/code/issue4/reproducibility_diagnoser.py \ + --self-test --output reports/issue4-order-demo.json + +# 单卡 NCCL 冒烟测试。它验证采集链路,不足以证明多 rank 可复现性。 +torchrun --standalone --nproc-per-node=1 \ + src/code/issue4/reproducibility_diagnoser.py \ + --runs 3 --calls 20 --output reports/issue4-single-gpu.json + +# 在 run 1 / call 3 / rank 0 注入一个 ULP,验证定位和误差统计链路。 +torchrun --standalone --nproc-per-node=1 \ + src/code/issue4/reproducibility_diagnoser.py \ + --runs 2 --calls 8 --inject-difference 1:3:0 --expect-difference \ + --output reports/issue4-injection.json +``` + +`--inject-difference` 只用于工具自测,不能当作 NCCL 不确定性的证据。 + +## 多卡验收运行 + +在 Linux 多卡机器上先运行默认配置,再固定关键选择项。每个命令内部都会对 +相同输入执行多轮对照,避免把输入随机性误判成通信差异。 + +```bash +# 默认选择逻辑 +torchrun --standalone --nproc-per-node=8 \ + src/code/issue4/reproducibility_diagnoser.py \ + --operation all-reduce --shape 1048576 --runs 5 --calls 100 \ + --output reports/all-reduce-default.json + +# 固定 Ring + Simple,关闭 NVLS,减少运行间算法/拓扑路径变化 +torchrun --standalone --nproc-per-node=8 \ + src/code/issue4/reproducibility_diagnoser.py \ + --operation all-reduce --shape 1048576 --runs 5 --calls 100 \ + --env NCCL_ALGO=Ring --env NCCL_PROTO=Simple --env NCCL_NVLS_ENABLE=0 \ + --output reports/all-reduce-ring-simple.json + +# Reduce-Scatter +torchrun --standalone --nproc-per-node=8 \ + src/code/issue4/reproducibility_diagnoser.py \ + --operation reduce-scatter --shape 131072 --runs 5 --calls 100 \ + --env NCCL_ALGO=Ring --env NCCL_PROTO=Simple \ + --output reports/reduce-scatter-ring-simple.json +``` + +建议逐项改变 `NCCL_ALGO`(Ring/Tree/PAT)、`NCCL_PROTO` +(Simple/LL/LL128)和 `NCCL_NVLS_ENABLE`,每个配置保存独立 JSON。跨机器复现时还应 +固定 GPU/rank 映射、NCCL/CUDA/驱动版本和拓扑文件。`torch.use_deterministic_algorithms` +只能约束 PyTorch 可控的本地算子,不能替代对 NCCL 算法、协议与拓扑的固定。 + +## 报告判读 + +- `status=bitwise-reproducible`:本次指定配置和运行窗口内未发现位级差异;这不是对所有 + 消息大小、拓扑和并发负载的普遍保证。 +- `status=difference-detected`:查看 `first_difference` 定位首次调用,再用 + `difference_evolution` 判断误差是否持续或放大。 +- `world_size=1`:只代表工具自检通过,不能形成 NCCL 多 rank 可复现性结论。 + diff --git a/src/code/issue4/reproducibility_diagnoser.py b/src/code/issue4/reproducibility_diagnoser.py new file mode 100644 index 0000000..c1c63a0 --- /dev/null +++ b/src/code/issue4/reproducibility_diagnoser.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python3 +"""Locate the first bitwise difference in repeated PyTorch collectives. + +The normal mode is intended to be launched with ``torchrun``. A small +dependency-free demo is also provided so that the comparison logic can be +checked before a CUDA/PyTorch environment is available. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import platform +import struct +import sys +import time +from datetime import timedelta +from pathlib import Path +from typing import Any + + +RELEVANT_ENV_VARS = ( + "NCCL_ALGO", + "NCCL_PROTO", + "NCCL_NVLS_ENABLE", + "NCCL_TOPO_FILE", + "NCCL_GRAPH_FILE", + "NCCL_P2P_LEVEL", + "NCCL_SHM_DISABLE", + "NCCL_NET", + "NCCL_DEBUG", + "CUBLAS_WORKSPACE_CONFIG", + "CUDA_VISIBLE_DEVICES", +) + + +def _float32(value: float) -> float: + """Round a Python float exactly as an IEEE-754 binary32 operation.""" + + return struct.unpack(" float: + return _float32(_float32(left) + _float32(right)) + + +def run_dependency_free_demo(output: Path | None) -> int: + """Show a reproducible reduction-order difference without PyTorch.""" + + values = [_float32(100_000_000.0), _float32(-100_000_000.0), _float32(1.0)] + left_associative = _add_float32(_add_float32(values[0], values[1]), values[2]) + right_associative = _add_float32(values[0], _add_float32(values[1], values[2])) + left_bits = struct.pack(" tuple[int, ...]: + try: + shape = tuple(int(part.strip()) for part in value.split(",") if part.strip()) + except ValueError as exc: + raise argparse.ArgumentTypeError("shape must be comma-separated positive integers") from exc + if not shape or any(dimension <= 0 for dimension in shape): + raise argparse.ArgumentTypeError("shape must contain positive integers") + return shape + + +def parse_injection(value: str) -> tuple[int, int, int]: + try: + run_index, call_index, rank = (int(part) for part in value.split(":")) + except (ValueError, TypeError) as exc: + raise argparse.ArgumentTypeError("injection must be RUN:CALL:RANK") from exc + if min(run_index, call_index, rank) < 0: + raise argparse.ArgumentTypeError("injection indices must be non-negative") + if run_index == 0: + raise argparse.ArgumentTypeError("run 0 is the reference; inject into run 1 or later") + return run_index, call_index, rank + + +def parse_env_assignment(value: str) -> tuple[str, str]: + if "=" not in value: + raise argparse.ArgumentTypeError("environment assignment must be KEY=VALUE") + key, assigned_value = value.split("=", 1) + if not key or not key.replace("_", "a").isalnum() or key[0].isdigit(): + raise argparse.ArgumentTypeError(f"invalid environment variable name: {key!r}") + return key, assigned_value + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Find the first bitwise difference across repeated collective calls." + ) + parser.add_argument("--self-test", action="store_true", help="run a dependency-free FP32 demo") + parser.add_argument("--backend", choices=("nccl", "gloo"), default="nccl") + parser.add_argument("--operation", choices=("all-reduce", "reduce-scatter"), default="all-reduce") + parser.add_argument("--shape", type=parse_shape, default=(4096,), help="output shape, e.g. 4096 or 32,128") + parser.add_argument("--dtype", choices=("float16", "bfloat16", "float32", "float64"), default="float32") + parser.add_argument("--runs", type=int, default=3, help="number of repeated runs (minimum 2)") + parser.add_argument("--calls", type=int, default=20, help="collective call points per run") + parser.add_argument("--warmup", type=int, default=5, help="unrecorded warmup collectives") + parser.add_argument("--seed", type=int, default=20260726) + parser.add_argument("--timeout-seconds", type=int, default=120) + parser.add_argument("--output", type=Path, default=Path("issue4-reproducibility-report.json")) + parser.add_argument( + "--env", + action="append", + type=parse_env_assignment, + default=[], + metavar="KEY=VALUE", + help="set an environment variable before initializing the process group", + ) + parser.add_argument( + "--inject-difference", + type=parse_injection, + metavar="RUN:CALL:RANK", + help="add one ULP after a collective to validate the detector", + ) + parser.add_argument( + "--expect-difference", + action="store_true", + help="return success only when a difference is found (useful with injection)", + ) + return parser + + +def _numel(shape: tuple[int, ...]) -> int: + return math.prod(shape) + + +def _tensor_bytes(tensor: Any) -> bytes: + # ``Tensor.numpy`` does not support every dtype (notably bfloat16), so view + # the storage as bytes and stay independent of NumPy. + raw_bytes = tensor.detach().contiguous().view(-1).view(__import__("torch").uint8).cpu() + return bytes(raw_bytes.tolist()) + + +def _digest(tensor: Any) -> str: + return hashlib.sha256(_tensor_bytes(tensor)).hexdigest() + + +def _make_input(torch: Any, args: argparse.Namespace, rank: int, call_index: int, world_size: int, device: Any) -> Any: + output_numel = _numel(args.shape) + input_numel = output_numel * (world_size if args.operation == "reduce-scatter" else 1) + generator = torch.Generator(device="cpu") + generator.manual_seed(args.seed + rank * 1_000_003 + call_index * 10_007) + + # Wide magnitudes and alternating signs make reduction-order changes easier + # to observe while remaining exactly repeatable across runs. + mantissa = torch.randn(input_numel, generator=generator, dtype=torch.float32) + exponent = torch.randint(-12, 13, (input_numel,), generator=generator) + values = torch.ldexp(mantissa, exponent).to(getattr(torch, args.dtype)) + return values.reshape((-1,) if args.operation == "reduce-scatter" else args.shape).to(device) + + +def _one_ulp_in_place(torch: Any, tensor: Any) -> None: + flat = tensor.reshape(-1) + direction = torch.full_like(flat[:1], float("inf")) + flat[:1] = torch.nextafter(flat[:1], direction) + + +def _difference(torch: Any, reference: Any, candidate: Any) -> dict[str, Any]: + element_size = reference.element_size() + reference_bytes = reference.detach().contiguous().view(torch.uint8).reshape(-1, element_size) + candidate_bytes = candidate.detach().contiguous().view(torch.uint8).reshape(-1, element_size) + element_mismatch = (reference_bytes != candidate_bytes).any(dim=1) + mismatch_count = int(element_mismatch.sum().item()) + if mismatch_count: + first_index = int(element_mismatch.nonzero(as_tuple=False)[0].item()) + delta = (candidate.reshape(-1).to(torch.float64) - reference.reshape(-1).to(torch.float64)).abs() + max_abs = float(delta.max().item()) + denominator = reference.reshape(-1).to(torch.float64).abs().clamp_min(torch.finfo(torch.float64).tiny) + max_rel = float((delta / denominator).max().item()) + else: + first_index = None + max_abs = 0.0 + max_rel = 0.0 + return { + "bitwise_equal": mismatch_count == 0, + "mismatched_elements": mismatch_count, + "first_mismatched_flat_index": first_index, + "max_abs_diff": max_abs, + "max_rel_diff": max_rel, + } + + +def _sync(torch: Any, device: Any) -> None: + if device.type == "cuda": + torch.cuda.synchronize(device) + + +def _nccl_version(torch: Any, backend: str) -> list[int] | str | None: + if backend != "nccl": + return None + version = torch.cuda.nccl.version() + if isinstance(version, tuple): + return list(version) + return str(version) + + +def run_collective_diagnosis(args: argparse.Namespace) -> int: + for key, value in args.env: + os.environ[key] = value + + try: + import torch + import torch.distributed as dist + except ImportError as exc: + print( + "PyTorch is required for collective mode. Run --self-test without dependencies, " + "or install a CUDA-enabled PyTorch build.", + file=sys.stderr, + ) + raise SystemExit(3) from exc + + if args.runs < 2 or args.calls < 1 or args.warmup < 0: + raise SystemExit("--runs must be >= 2, --calls >= 1, and --warmup >= 0") + if not dist.is_available(): + raise SystemExit("torch.distributed is unavailable in this PyTorch build") + if args.backend == "nccl" and not dist.is_nccl_available(): + raise SystemExit("this PyTorch build does not include NCCL") + if args.backend == "nccl" and not torch.cuda.is_available(): + raise SystemExit("NCCL mode requires a CUDA device") + + local_rank = int(os.environ.get("LOCAL_RANK", "0")) + if args.backend == "nccl": + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + else: + device = torch.device("cpu") + + init_kwargs = { + "backend": args.backend, + "init_method": "env://", + "timeout": timedelta(seconds=args.timeout_seconds), + } + if args.backend == "nccl": + # Eagerly bind the process group to its CUDA device. This also makes + # barrier device selection explicit on recent PyTorch releases. + init_kwargs["device_id"] = device + dist.init_process_group(**init_kwargs) + rank = dist.get_rank() + world_size = dist.get_world_size() + injection = args.inject_difference + if injection is not None and injection[2] >= world_size: + raise SystemExit(f"injection rank {injection[2]} is outside world size {world_size}") + + torch.use_deterministic_algorithms(True) + baselines: list[Any] = [] + local_records: list[dict[str, Any]] = [] + + def execute(call_index: int) -> Any: + source = _make_input(torch, args, rank, call_index, world_size, device) + if args.operation == "all-reduce": + output = source + dist.all_reduce(output, op=dist.ReduceOp.SUM) + else: + output = torch.empty(args.shape, dtype=getattr(torch, args.dtype), device=device) + reduce_scatter = getattr(dist, "reduce_scatter_single", dist.reduce_scatter_tensor) + reduce_scatter(output, source, op=dist.ReduceOp.SUM) + return output + + try: + for warmup_index in range(args.warmup): + execute(-(warmup_index + 1)) + _sync(torch, device) + dist.barrier() + + for run_index in range(args.runs): + for call_index in range(args.calls): + dist.barrier() + _sync(torch, device) + started = time.perf_counter_ns() + output = execute(call_index) + _sync(torch, device) + elapsed_us = (time.perf_counter_ns() - started) / 1_000.0 + + if injection == (run_index, call_index, rank): + _one_ulp_in_place(torch, output) + + record: dict[str, Any] = { + "rank": rank, + "run": run_index, + "call": call_index, + "elapsed_us": elapsed_us, + "digest": _digest(output), + } + if run_index == 0: + baselines.append(output.detach().clone()) + record.update( + bitwise_equal=True, + mismatched_elements=0, + first_mismatched_flat_index=None, + max_abs_diff=0.0, + max_rel_diff=0.0, + ) + else: + record.update(_difference(torch, baselines[call_index], output)) + local_records.append(record) + + gathered: list[Any] = [None] * world_size + dist.all_gather_object(gathered, local_records) + if rank != 0: + return 0 + + records = [record for rank_records in gathered for record in rank_records] + differences = [record for record in records if not record["bitwise_equal"]] + differences.sort(key=lambda record: (record["run"], record["call"], record["rank"])) + first_difference = differences[0] if differences else None + report = { + "schema_version": 1, + "mode": "collective-run-to-run", + "status": "difference-detected" if first_difference else "bitwise-reproducible", + "configuration": { + "backend": args.backend, + "operation": args.operation, + "shape": args.shape, + "dtype": args.dtype, + "runs": args.runs, + "calls": args.calls, + "warmup": args.warmup, + "seed": args.seed, + "world_size": world_size, + "injected_difference": injection, + }, + "environment": { + "python": platform.python_version(), + "pytorch": torch.__version__, + "cuda_runtime": torch.version.cuda, + "nccl": _nccl_version(torch, args.backend), + "devices": [torch.cuda.get_device_name(index) for index in range(torch.cuda.device_count())], + "variables": {key: os.environ[key] for key in RELEVANT_ENV_VARS if key in os.environ}, + }, + "first_difference": first_difference, + "difference_evolution": differences, + "records": sorted(records, key=lambda record: (record["run"], record["call"], record["rank"])), + "limitations": ( + "world_size=1 validates the tool path but cannot expose reduction-order nondeterminism; " + "use at least two GPUs for an NCCL reproducibility conclusion." + if world_size == 1 + else None + ), + } + rendered = json.dumps(report, ensure_ascii=False, indent=2) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered + "\n", encoding="utf-8") + print(rendered) + print(f"Report written to {args.output}", file=sys.stderr) + + detected = first_difference is not None + if args.expect_difference: + return 0 if detected else 1 + return 0 + finally: + if dist.is_initialized(): + dist.destroy_process_group() + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.self_test: + return run_dependency_free_demo(args.output) + return run_collective_diagnosis(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/code/issue4/results/single_gpu_all_reduce.json b/src/code/issue4/results/single_gpu_all_reduce.json new file mode 100644 index 0000000..800fffa --- /dev/null +++ b/src/code/issue4/results/single_gpu_all_reduce.json @@ -0,0 +1,758 @@ +{ + "schema_version": 1, + "mode": "collective-run-to-run", + "status": "bitwise-reproducible", + "configuration": { + "backend": "nccl", + "operation": "all-reduce", + "shape": [ + 4096 + ], + "dtype": "float32", + "runs": 3, + "calls": 20, + "warmup": 5, + "seed": 20260726, + "world_size": 1, + "injected_difference": null + }, + "environment": { + "python": "3.12.3", + "pytorch": "2.13.0+cu130", + "cuda_runtime": "13.0", + "nccl": [ + 2, + 29, + 7 + ], + "devices": [ + "NVIDIA GeForce RTX 5060 Ti" + ], + "variables": {} + }, + "first_difference": null, + "difference_evolution": [], + "records": [ + { + "rank": 0, + "run": 0, + "call": 0, + "elapsed_us": 152.377, + "digest": "8bf2ed82379ff6ab9d4d585296fc7d2a2054fbdd90f78a0377cf941f64f1d27a", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 1, + "elapsed_us": 651.968, + "digest": "f1703a630e18beee10af4620967d2c9f3612681d95872db9c973f6e54cc4051e", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 2, + "elapsed_us": 163.24, + "digest": "2e66c761278926b5074c15d948c6ddff1ea84a0634df7258bb04c52a43b33c95", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 3, + "elapsed_us": 118.761, + "digest": "9f925898775c7b3c31a7432b9f4e7ad5cefd59579c8c29685b01a41ac2f9dfcf", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 4, + "elapsed_us": 131.658, + "digest": "2d45d208dff8d2c2fb61dbb76dcfe48b2ea76a1edb60b8da6d9279beec06548c", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 5, + "elapsed_us": 119.499, + "digest": "f3e5f6003ef8b450066dd062d9ac394b97d99eb1534ed61dd467acc9e5d9d713", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 6, + "elapsed_us": 109.53, + "digest": "99f35075460016cd837582a5043997d517474c17a1a179ab93d26d3a832a9d20", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 7, + "elapsed_us": 120.788, + "digest": "9d0beb8855b2f8728d5812d8b874260ac9f31c1e9a673f65cfa98527a731c2cf", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 8, + "elapsed_us": 109.101, + "digest": "25786dbc53b79a7cf2e3cb0341e7069eb8159a69eb29eeb9735fd8e55cfc7ddb", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 9, + "elapsed_us": 110.75, + "digest": "9d98ae2af448a2f6b96416721ab43601afeb7d583f00114f0ad3c7fbd70c037b", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 10, + "elapsed_us": 105.159, + "digest": "6747ae860bdad9ac8ca1266343c3576621139bcf23b4f5ddfe58bf682da50bee", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 11, + "elapsed_us": 207.399, + "digest": "9b701e4737101966e5a6ebf6eeb23050e974b355d48353e0bdb3e2868523f7ad", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 12, + "elapsed_us": 138.28, + "digest": "a5f8a65b25a5231b1ee35d87205e08c19fa08210bee36ccfcaf28ae83bf1a52c", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 13, + "elapsed_us": 122.674, + "digest": "abc1a4c919ef5e07d2cd532009548824cfa5fedae39735304d2019ed9a600f69", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 14, + "elapsed_us": 143.096, + "digest": "719255d80e62231571ab0655b3eb953e87ed2dd1d11f8423b2f1ccdb3ac02424", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 15, + "elapsed_us": 117.93, + "digest": "c60210cd306c00bc9679ebd837857b4ec58dfb403d6ea22e8ef147021c366e2f", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 16, + "elapsed_us": 110.294, + "digest": "e7ad4b0df32fff1c5799b4bf34d4380f7ad340e21c3a6c19eb2cf4b8fc25b9b1", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 17, + "elapsed_us": 119.54, + "digest": "d73f1c25b5e020af25f20a75af8019e6921dfa533f6fc9a00049b62c8452a434", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 18, + "elapsed_us": 127.141, + "digest": "4a55e71cd33f4392efaa99a6bdef8218dad2c4f90f45975fedb9eca3f04c2846", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 19, + "elapsed_us": 110.264, + "digest": "18a9adf662c5a45c00415d5b002f153992a13e90208c86844e5e873e8fe5fcd6", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 0, + "elapsed_us": 128.798, + "digest": "8bf2ed82379ff6ab9d4d585296fc7d2a2054fbdd90f78a0377cf941f64f1d27a", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 1, + "elapsed_us": 332.942, + "digest": "f1703a630e18beee10af4620967d2c9f3612681d95872db9c973f6e54cc4051e", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 2, + "elapsed_us": 140.948, + "digest": "2e66c761278926b5074c15d948c6ddff1ea84a0634df7258bb04c52a43b33c95", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 3, + "elapsed_us": 131.729, + "digest": "9f925898775c7b3c31a7432b9f4e7ad5cefd59579c8c29685b01a41ac2f9dfcf", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 4, + "elapsed_us": 116.205, + "digest": "2d45d208dff8d2c2fb61dbb76dcfe48b2ea76a1edb60b8da6d9279beec06548c", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 5, + "elapsed_us": 110.944, + "digest": "f3e5f6003ef8b450066dd062d9ac394b97d99eb1534ed61dd467acc9e5d9d713", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 6, + "elapsed_us": 106.006, + "digest": "99f35075460016cd837582a5043997d517474c17a1a179ab93d26d3a832a9d20", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 7, + "elapsed_us": 105.617, + "digest": "9d0beb8855b2f8728d5812d8b874260ac9f31c1e9a673f65cfa98527a731c2cf", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 8, + "elapsed_us": 111.578, + "digest": "25786dbc53b79a7cf2e3cb0341e7069eb8159a69eb29eeb9735fd8e55cfc7ddb", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 9, + "elapsed_us": 107.087, + "digest": "9d98ae2af448a2f6b96416721ab43601afeb7d583f00114f0ad3c7fbd70c037b", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 10, + "elapsed_us": 286.906, + "digest": "6747ae860bdad9ac8ca1266343c3576621139bcf23b4f5ddfe58bf682da50bee", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 11, + "elapsed_us": 289.451, + "digest": "9b701e4737101966e5a6ebf6eeb23050e974b355d48353e0bdb3e2868523f7ad", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 12, + "elapsed_us": 139.178, + "digest": "a5f8a65b25a5231b1ee35d87205e08c19fa08210bee36ccfcaf28ae83bf1a52c", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 13, + "elapsed_us": 138.878, + "digest": "abc1a4c919ef5e07d2cd532009548824cfa5fedae39735304d2019ed9a600f69", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 14, + "elapsed_us": 128.863, + "digest": "719255d80e62231571ab0655b3eb953e87ed2dd1d11f8423b2f1ccdb3ac02424", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 15, + "elapsed_us": 124.103, + "digest": "c60210cd306c00bc9679ebd837857b4ec58dfb403d6ea22e8ef147021c366e2f", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 16, + "elapsed_us": 139.677, + "digest": "e7ad4b0df32fff1c5799b4bf34d4380f7ad340e21c3a6c19eb2cf4b8fc25b9b1", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 17, + "elapsed_us": 126.93, + "digest": "d73f1c25b5e020af25f20a75af8019e6921dfa533f6fc9a00049b62c8452a434", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 18, + "elapsed_us": 118.136, + "digest": "4a55e71cd33f4392efaa99a6bdef8218dad2c4f90f45975fedb9eca3f04c2846", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 19, + "elapsed_us": 123.165, + "digest": "18a9adf662c5a45c00415d5b002f153992a13e90208c86844e5e873e8fe5fcd6", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 0, + "elapsed_us": 162.249, + "digest": "8bf2ed82379ff6ab9d4d585296fc7d2a2054fbdd90f78a0377cf941f64f1d27a", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 1, + "elapsed_us": 133.124, + "digest": "f1703a630e18beee10af4620967d2c9f3612681d95872db9c973f6e54cc4051e", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 2, + "elapsed_us": 140.581, + "digest": "2e66c761278926b5074c15d948c6ddff1ea84a0634df7258bb04c52a43b33c95", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 3, + "elapsed_us": 122.519, + "digest": "9f925898775c7b3c31a7432b9f4e7ad5cefd59579c8c29685b01a41ac2f9dfcf", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 4, + "elapsed_us": 123.717, + "digest": "2d45d208dff8d2c2fb61dbb76dcfe48b2ea76a1edb60b8da6d9279beec06548c", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 5, + "elapsed_us": 127.113, + "digest": "f3e5f6003ef8b450066dd062d9ac394b97d99eb1534ed61dd467acc9e5d9d713", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 6, + "elapsed_us": 121.961, + "digest": "99f35075460016cd837582a5043997d517474c17a1a179ab93d26d3a832a9d20", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 7, + "elapsed_us": 123.355, + "digest": "9d0beb8855b2f8728d5812d8b874260ac9f31c1e9a673f65cfa98527a731c2cf", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 8, + "elapsed_us": 124.101, + "digest": "25786dbc53b79a7cf2e3cb0341e7069eb8159a69eb29eeb9735fd8e55cfc7ddb", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 9, + "elapsed_us": 122.622, + "digest": "9d98ae2af448a2f6b96416721ab43601afeb7d583f00114f0ad3c7fbd70c037b", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 10, + "elapsed_us": 125.389, + "digest": "6747ae860bdad9ac8ca1266343c3576621139bcf23b4f5ddfe58bf682da50bee", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 11, + "elapsed_us": 117.855, + "digest": "9b701e4737101966e5a6ebf6eeb23050e974b355d48353e0bdb3e2868523f7ad", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 12, + "elapsed_us": 124.239, + "digest": "a5f8a65b25a5231b1ee35d87205e08c19fa08210bee36ccfcaf28ae83bf1a52c", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 13, + "elapsed_us": 151.87, + "digest": "abc1a4c919ef5e07d2cd532009548824cfa5fedae39735304d2019ed9a600f69", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 14, + "elapsed_us": 141.774, + "digest": "719255d80e62231571ab0655b3eb953e87ed2dd1d11f8423b2f1ccdb3ac02424", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 15, + "elapsed_us": 142.962, + "digest": "c60210cd306c00bc9679ebd837857b4ec58dfb403d6ea22e8ef147021c366e2f", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 16, + "elapsed_us": 233.637, + "digest": "e7ad4b0df32fff1c5799b4bf34d4380f7ad340e21c3a6c19eb2cf4b8fc25b9b1", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 17, + "elapsed_us": 159.849, + "digest": "d73f1c25b5e020af25f20a75af8019e6921dfa533f6fc9a00049b62c8452a434", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 18, + "elapsed_us": 146.017, + "digest": "4a55e71cd33f4392efaa99a6bdef8218dad2c4f90f45975fedb9eca3f04c2846", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 19, + "elapsed_us": 145.195, + "digest": "18a9adf662c5a45c00415d5b002f153992a13e90208c86844e5e873e8fe5fcd6", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + } + ], + "limitations": "world_size=1 validates the tool path but cannot expose reduction-order nondeterminism; use at least two GPUs for an NCCL reproducibility conclusion." +} diff --git a/src/code/issue4/results/single_gpu_injection.json b/src/code/issue4/results/single_gpu_injection.json new file mode 100644 index 0000000..b37adc4 --- /dev/null +++ b/src/code/issue4/results/single_gpu_injection.json @@ -0,0 +1,258 @@ +{ + "schema_version": 1, + "mode": "collective-run-to-run", + "status": "difference-detected", + "configuration": { + "backend": "nccl", + "operation": "all-reduce", + "shape": [ + 4096 + ], + "dtype": "float32", + "runs": 2, + "calls": 8, + "warmup": 5, + "seed": 20260726, + "world_size": 1, + "injected_difference": [ + 1, + 3, + 0 + ] + }, + "environment": { + "python": "3.12.3", + "pytorch": "2.13.0+cu130", + "cuda_runtime": "13.0", + "nccl": [ + 2, + 29, + 7 + ], + "devices": [ + "NVIDIA GeForce RTX 5060 Ti" + ], + "variables": {} + }, + "first_difference": { + "rank": 0, + "run": 1, + "call": 3, + "elapsed_us": 131.624, + "digest": "0da7416a1e05f7b027da291cf236c70e86cd642a3c7450abbd27bce56e1c2064", + "bitwise_equal": false, + "mismatched_elements": 1, + "first_mismatched_flat_index": 0, + "max_abs_diff": 3.814697265625e-06, + "max_rel_diff": 9.435530698593483e-08 + }, + "difference_evolution": [ + { + "rank": 0, + "run": 1, + "call": 3, + "elapsed_us": 131.624, + "digest": "0da7416a1e05f7b027da291cf236c70e86cd642a3c7450abbd27bce56e1c2064", + "bitwise_equal": false, + "mismatched_elements": 1, + "first_mismatched_flat_index": 0, + "max_abs_diff": 3.814697265625e-06, + "max_rel_diff": 9.435530698593483e-08 + } + ], + "records": [ + { + "rank": 0, + "run": 0, + "call": 0, + "elapsed_us": 219.829, + "digest": "8bf2ed82379ff6ab9d4d585296fc7d2a2054fbdd90f78a0377cf941f64f1d27a", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 1, + "elapsed_us": 152.522, + "digest": "f1703a630e18beee10af4620967d2c9f3612681d95872db9c973f6e54cc4051e", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 2, + "elapsed_us": 140.705, + "digest": "2e66c761278926b5074c15d948c6ddff1ea84a0634df7258bb04c52a43b33c95", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 3, + "elapsed_us": 133.691, + "digest": "9f925898775c7b3c31a7432b9f4e7ad5cefd59579c8c29685b01a41ac2f9dfcf", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 4, + "elapsed_us": 138.692, + "digest": "2d45d208dff8d2c2fb61dbb76dcfe48b2ea76a1edb60b8da6d9279beec06548c", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 5, + "elapsed_us": 141.722, + "digest": "f3e5f6003ef8b450066dd062d9ac394b97d99eb1534ed61dd467acc9e5d9d713", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 6, + "elapsed_us": 164.116, + "digest": "99f35075460016cd837582a5043997d517474c17a1a179ab93d26d3a832a9d20", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 7, + "elapsed_us": 194.084, + "digest": "9d0beb8855b2f8728d5812d8b874260ac9f31c1e9a673f65cfa98527a731c2cf", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 0, + "elapsed_us": 149.209, + "digest": "8bf2ed82379ff6ab9d4d585296fc7d2a2054fbdd90f78a0377cf941f64f1d27a", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 1, + "elapsed_us": 315.298, + "digest": "f1703a630e18beee10af4620967d2c9f3612681d95872db9c973f6e54cc4051e", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 2, + "elapsed_us": 154.95, + "digest": "2e66c761278926b5074c15d948c6ddff1ea84a0634df7258bb04c52a43b33c95", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 3, + "elapsed_us": 131.624, + "digest": "0da7416a1e05f7b027da291cf236c70e86cd642a3c7450abbd27bce56e1c2064", + "bitwise_equal": false, + "mismatched_elements": 1, + "first_mismatched_flat_index": 0, + "max_abs_diff": 3.814697265625e-06, + "max_rel_diff": 9.435530698593483e-08 + }, + { + "rank": 0, + "run": 1, + "call": 4, + "elapsed_us": 317.9, + "digest": "2d45d208dff8d2c2fb61dbb76dcfe48b2ea76a1edb60b8da6d9279beec06548c", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 5, + "elapsed_us": 138.857, + "digest": "f3e5f6003ef8b450066dd062d9ac394b97d99eb1534ed61dd467acc9e5d9d713", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 6, + "elapsed_us": 379.83, + "digest": "99f35075460016cd837582a5043997d517474c17a1a179ab93d26d3a832a9d20", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 7, + "elapsed_us": 154.629, + "digest": "9d0beb8855b2f8728d5812d8b874260ac9f31c1e9a673f65cfa98527a731c2cf", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + } + ], + "limitations": "world_size=1 validates the tool path but cannot expose reduction-order nondeterminism; use at least two GPUs for an NCCL reproducibility conclusion." +} diff --git a/src/code/issue4/results/single_gpu_reduce_scatter.json b/src/code/issue4/results/single_gpu_reduce_scatter.json new file mode 100644 index 0000000..b0db0fb --- /dev/null +++ b/src/code/issue4/results/single_gpu_reduce_scatter.json @@ -0,0 +1,758 @@ +{ + "schema_version": 1, + "mode": "collective-run-to-run", + "status": "bitwise-reproducible", + "configuration": { + "backend": "nccl", + "operation": "reduce-scatter", + "shape": [ + 4096 + ], + "dtype": "float32", + "runs": 3, + "calls": 20, + "warmup": 5, + "seed": 20260726, + "world_size": 1, + "injected_difference": null + }, + "environment": { + "python": "3.12.3", + "pytorch": "2.13.0+cu130", + "cuda_runtime": "13.0", + "nccl": [ + 2, + 29, + 7 + ], + "devices": [ + "NVIDIA GeForce RTX 5060 Ti" + ], + "variables": {} + }, + "first_difference": null, + "difference_evolution": [], + "records": [ + { + "rank": 0, + "run": 0, + "call": 0, + "elapsed_us": 166.742, + "digest": "8bf2ed82379ff6ab9d4d585296fc7d2a2054fbdd90f78a0377cf941f64f1d27a", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 1, + "elapsed_us": 155.998, + "digest": "f1703a630e18beee10af4620967d2c9f3612681d95872db9c973f6e54cc4051e", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 2, + "elapsed_us": 137.768, + "digest": "2e66c761278926b5074c15d948c6ddff1ea84a0634df7258bb04c52a43b33c95", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 3, + "elapsed_us": 135.448, + "digest": "9f925898775c7b3c31a7432b9f4e7ad5cefd59579c8c29685b01a41ac2f9dfcf", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 4, + "elapsed_us": 149.629, + "digest": "2d45d208dff8d2c2fb61dbb76dcfe48b2ea76a1edb60b8da6d9279beec06548c", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 5, + "elapsed_us": 133.315, + "digest": "f3e5f6003ef8b450066dd062d9ac394b97d99eb1534ed61dd467acc9e5d9d713", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 6, + "elapsed_us": 141.485, + "digest": "99f35075460016cd837582a5043997d517474c17a1a179ab93d26d3a832a9d20", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 7, + "elapsed_us": 219.896, + "digest": "9d0beb8855b2f8728d5812d8b874260ac9f31c1e9a673f65cfa98527a731c2cf", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 8, + "elapsed_us": 138.526, + "digest": "25786dbc53b79a7cf2e3cb0341e7069eb8159a69eb29eeb9735fd8e55cfc7ddb", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 9, + "elapsed_us": 566.182, + "digest": "9d98ae2af448a2f6b96416721ab43601afeb7d583f00114f0ad3c7fbd70c037b", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 10, + "elapsed_us": 397.863, + "digest": "6747ae860bdad9ac8ca1266343c3576621139bcf23b4f5ddfe58bf682da50bee", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 11, + "elapsed_us": 132.985, + "digest": "9b701e4737101966e5a6ebf6eeb23050e974b355d48353e0bdb3e2868523f7ad", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 12, + "elapsed_us": 124.898, + "digest": "a5f8a65b25a5231b1ee35d87205e08c19fa08210bee36ccfcaf28ae83bf1a52c", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 13, + "elapsed_us": 123.902, + "digest": "abc1a4c919ef5e07d2cd532009548824cfa5fedae39735304d2019ed9a600f69", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 14, + "elapsed_us": 128.182, + "digest": "719255d80e62231571ab0655b3eb953e87ed2dd1d11f8423b2f1ccdb3ac02424", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 15, + "elapsed_us": 124.985, + "digest": "c60210cd306c00bc9679ebd837857b4ec58dfb403d6ea22e8ef147021c366e2f", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 16, + "elapsed_us": 126.661, + "digest": "e7ad4b0df32fff1c5799b4bf34d4380f7ad340e21c3a6c19eb2cf4b8fc25b9b1", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 17, + "elapsed_us": 127.71, + "digest": "d73f1c25b5e020af25f20a75af8019e6921dfa533f6fc9a00049b62c8452a434", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 18, + "elapsed_us": 140.909, + "digest": "4a55e71cd33f4392efaa99a6bdef8218dad2c4f90f45975fedb9eca3f04c2846", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 0, + "call": 19, + "elapsed_us": 129.103, + "digest": "18a9adf662c5a45c00415d5b002f153992a13e90208c86844e5e873e8fe5fcd6", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 0, + "elapsed_us": 122.116, + "digest": "8bf2ed82379ff6ab9d4d585296fc7d2a2054fbdd90f78a0377cf941f64f1d27a", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 1, + "elapsed_us": 450.84, + "digest": "f1703a630e18beee10af4620967d2c9f3612681d95872db9c973f6e54cc4051e", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 2, + "elapsed_us": 155.373, + "digest": "2e66c761278926b5074c15d948c6ddff1ea84a0634df7258bb04c52a43b33c95", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 3, + "elapsed_us": 140.068, + "digest": "9f925898775c7b3c31a7432b9f4e7ad5cefd59579c8c29685b01a41ac2f9dfcf", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 4, + "elapsed_us": 138.157, + "digest": "2d45d208dff8d2c2fb61dbb76dcfe48b2ea76a1edb60b8da6d9279beec06548c", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 5, + "elapsed_us": 138.928, + "digest": "f3e5f6003ef8b450066dd062d9ac394b97d99eb1534ed61dd467acc9e5d9d713", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 6, + "elapsed_us": 133.923, + "digest": "99f35075460016cd837582a5043997d517474c17a1a179ab93d26d3a832a9d20", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 7, + "elapsed_us": 131.536, + "digest": "9d0beb8855b2f8728d5812d8b874260ac9f31c1e9a673f65cfa98527a731c2cf", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 8, + "elapsed_us": 204.887, + "digest": "25786dbc53b79a7cf2e3cb0341e7069eb8159a69eb29eeb9735fd8e55cfc7ddb", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 9, + "elapsed_us": 133.913, + "digest": "9d98ae2af448a2f6b96416721ab43601afeb7d583f00114f0ad3c7fbd70c037b", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 10, + "elapsed_us": 134.968, + "digest": "6747ae860bdad9ac8ca1266343c3576621139bcf23b4f5ddfe58bf682da50bee", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 11, + "elapsed_us": 132.676, + "digest": "9b701e4737101966e5a6ebf6eeb23050e974b355d48353e0bdb3e2868523f7ad", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 12, + "elapsed_us": 136.39, + "digest": "a5f8a65b25a5231b1ee35d87205e08c19fa08210bee36ccfcaf28ae83bf1a52c", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 13, + "elapsed_us": 133.545, + "digest": "abc1a4c919ef5e07d2cd532009548824cfa5fedae39735304d2019ed9a600f69", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 14, + "elapsed_us": 130.62, + "digest": "719255d80e62231571ab0655b3eb953e87ed2dd1d11f8423b2f1ccdb3ac02424", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 15, + "elapsed_us": 131.918, + "digest": "c60210cd306c00bc9679ebd837857b4ec58dfb403d6ea22e8ef147021c366e2f", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 16, + "elapsed_us": 130.75, + "digest": "e7ad4b0df32fff1c5799b4bf34d4380f7ad340e21c3a6c19eb2cf4b8fc25b9b1", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 17, + "elapsed_us": 133.307, + "digest": "d73f1c25b5e020af25f20a75af8019e6921dfa533f6fc9a00049b62c8452a434", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 18, + "elapsed_us": 134.691, + "digest": "4a55e71cd33f4392efaa99a6bdef8218dad2c4f90f45975fedb9eca3f04c2846", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 1, + "call": 19, + "elapsed_us": 187.364, + "digest": "18a9adf662c5a45c00415d5b002f153992a13e90208c86844e5e873e8fe5fcd6", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 0, + "elapsed_us": 148.723, + "digest": "8bf2ed82379ff6ab9d4d585296fc7d2a2054fbdd90f78a0377cf941f64f1d27a", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 1, + "elapsed_us": 127.31, + "digest": "f1703a630e18beee10af4620967d2c9f3612681d95872db9c973f6e54cc4051e", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 2, + "elapsed_us": 122.428, + "digest": "2e66c761278926b5074c15d948c6ddff1ea84a0634df7258bb04c52a43b33c95", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 3, + "elapsed_us": 121.789, + "digest": "9f925898775c7b3c31a7432b9f4e7ad5cefd59579c8c29685b01a41ac2f9dfcf", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 4, + "elapsed_us": 135.499, + "digest": "2d45d208dff8d2c2fb61dbb76dcfe48b2ea76a1edb60b8da6d9279beec06548c", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 5, + "elapsed_us": 241.729, + "digest": "f3e5f6003ef8b450066dd062d9ac394b97d99eb1534ed61dd467acc9e5d9d713", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 6, + "elapsed_us": 142.247, + "digest": "99f35075460016cd837582a5043997d517474c17a1a179ab93d26d3a832a9d20", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 7, + "elapsed_us": 125.003, + "digest": "9d0beb8855b2f8728d5812d8b874260ac9f31c1e9a673f65cfa98527a731c2cf", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 8, + "elapsed_us": 146.66, + "digest": "25786dbc53b79a7cf2e3cb0341e7069eb8159a69eb29eeb9735fd8e55cfc7ddb", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 9, + "elapsed_us": 122.307, + "digest": "9d98ae2af448a2f6b96416721ab43601afeb7d583f00114f0ad3c7fbd70c037b", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 10, + "elapsed_us": 125.22, + "digest": "6747ae860bdad9ac8ca1266343c3576621139bcf23b4f5ddfe58bf682da50bee", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 11, + "elapsed_us": 2938.514, + "digest": "9b701e4737101966e5a6ebf6eeb23050e974b355d48353e0bdb3e2868523f7ad", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 12, + "elapsed_us": 148.872, + "digest": "a5f8a65b25a5231b1ee35d87205e08c19fa08210bee36ccfcaf28ae83bf1a52c", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 13, + "elapsed_us": 140.968, + "digest": "abc1a4c919ef5e07d2cd532009548824cfa5fedae39735304d2019ed9a600f69", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 14, + "elapsed_us": 150.314, + "digest": "719255d80e62231571ab0655b3eb953e87ed2dd1d11f8423b2f1ccdb3ac02424", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 15, + "elapsed_us": 146.666, + "digest": "c60210cd306c00bc9679ebd837857b4ec58dfb403d6ea22e8ef147021c366e2f", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 16, + "elapsed_us": 127.412, + "digest": "e7ad4b0df32fff1c5799b4bf34d4380f7ad340e21c3a6c19eb2cf4b8fc25b9b1", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 17, + "elapsed_us": 128.504, + "digest": "d73f1c25b5e020af25f20a75af8019e6921dfa533f6fc9a00049b62c8452a434", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 18, + "elapsed_us": 121.786, + "digest": "4a55e71cd33f4392efaa99a6bdef8218dad2c4f90f45975fedb9eca3f04c2846", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + }, + { + "rank": 0, + "run": 2, + "call": 19, + "elapsed_us": 122.678, + "digest": "18a9adf662c5a45c00415d5b002f153992a13e90208c86844e5e873e8fe5fcd6", + "bitwise_equal": true, + "mismatched_elements": 0, + "first_mismatched_flat_index": null, + "max_abs_diff": 0.0, + "max_rel_diff": 0.0 + } + ], + "limitations": "world_size=1 validates the tool path but cannot expose reduction-order nondeterminism; use at least two GPUs for an NCCL reproducibility conclusion." +} diff --git a/src/code/issue4/test_reproducibility_diagnoser.py b/src/code/issue4/test_reproducibility_diagnoser.py new file mode 100644 index 0000000..8feafb2 --- /dev/null +++ b/src/code/issue4/test_reproducibility_diagnoser.py @@ -0,0 +1,31 @@ +import importlib.util +import json +from pathlib import Path + + +MODULE_PATH = Path(__file__).with_name("reproducibility_diagnoser.py") +SPEC = importlib.util.spec_from_file_location("reproducibility_diagnoser", MODULE_PATH) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(MODULE) + + +def test_reduction_order_demo_detects_bitwise_difference(tmp_path): + report_path = tmp_path / "demo.json" + + assert MODULE.run_dependency_free_demo(report_path) == 0 + + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["detected_difference"] is True + assert report["runs"][0]["value"] == 1.0 + assert report["runs"][1]["value"] == 0.0 + assert report["max_abs_diff"] == 1.0 + + +def test_shape_parser(): + assert MODULE.parse_shape("32, 128") == (32, 128) + + +def test_injection_parser(): + assert MODULE.parse_injection("1:3:0") == (1, 3, 0) +