diff --git a/tests/integration/defs/accuracy/accuracy_core.py b/tests/integration/defs/accuracy/accuracy_core.py index fedbfddbab90..605aadd80590 100644 --- a/tests/integration/defs/accuracy/accuracy_core.py +++ b/tests/integration/defs/accuracy/accuracy_core.py @@ -69,6 +69,9 @@ def compute_threshold(num_samples: int, return ref_accuracy - z_alpha * scale +STDERR_MARGIN_SIGMAS = 2.0 + + @dataclass(slots=True) class HypothesisTestingParams: ref_accuracy: float @@ -78,10 +81,19 @@ class HypothesisTestingParams: beta: float = 0.2 sigma: float = 50.0 higher_is_better: bool = True + # lm-eval-style per-metric stderr. When set, ref_accuracy is treated as an + # anchor and the strict floor is offset by STDERR_MARGIN_SIGMAS * stderr so + # single-run noise does not trip the outer assertion (nvbugs/6506920). + stderr: Optional[float] = None + ref_accuracy_anchor: float = field(init=False) theta: float = field(init=False) threshold: float = field(init=False) def __post_init__(self) -> None: + self.ref_accuracy_anchor = self.ref_accuracy + if self.stderr is not None and self.stderr > 0: + sign = -1 if self.higher_is_better else 1 + self.ref_accuracy += sign * STDERR_MARGIN_SIGMAS * self.stderr self.theta = compute_theta(self.num_samples, sigma=self.sigma, alpha=self.alpha, @@ -95,6 +107,13 @@ def __post_init__(self) -> None: def report(self, accuracy: Optional[float] = None) -> str: metric_name = self.metric_name.upper() + if self.stderr is not None and self.stderr > 0: + stderr_line = ( + f"\nAnchor {self.metric_name}: {self.ref_accuracy_anchor:.3f}" + f" (adjusted by {STDERR_MARGIN_SIGMAS:.1f}sigma * stderr={self.stderr:.3f})" + ) + else: + stderr_line = "" report = f"""=========================================================== = {metric_name} HYPOTHESIS TESTING =========================================================== @@ -104,7 +123,7 @@ def report(self, accuracy: Optional[float] = None) -> str: #Samples: {self.num_samples} Higher is better: {self.higher_is_better} Theta (Minimum detectable effect): {self.theta:.3f} -Reference {self.metric_name}: {self.ref_accuracy:.3f} +Reference {self.metric_name}: {self.ref_accuracy:.3f}{stderr_line} Threshold: {self.threshold:.3f} ===========================================================""" if accuracy is not None: @@ -190,7 +209,8 @@ def get_hypothesis_testing_params(self, sigma=entry.get("sigma", self.SIGMA), num_samples=entry.get("num_samples", self.NUM_SAMPLES), higher_is_better=entry.get("higher_is_better", - self.HIGHER_IS_BETTER)) + self.HIGHER_IS_BETTER), + stderr=entry.get("stderr")) def evaluate(self, llm: Union[PyTorchLLM, AutoDeployLLM], diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml index ef32f74f1ba0..942e542251be 100644 --- a/tests/integration/defs/accuracy/references/gsm8k.yaml +++ b/tests/integration/defs/accuracy/references/gsm8k.yaml @@ -156,11 +156,16 @@ deepseek-ai/DeepSeek-V4-Pro: spec_dec_algo: MTP accuracy: 96.0 # Full GSM8K DEP8 DSpark path: TP=8, EP=8, attention DP, MegaMoE DeepGEMM, - # and max_draft_len=5. Measured 96.475 on 1319 samples; floor set to 96.0 - # for run-to-run margin. + # and max_draft_len=5. Author-anchor 96.475 on 1319 samples with lm-eval + # reporting per-metric stderr ~0.55; observed spread across runs + # 95.94..96.55 tracks that stderr. accuracy_core subtracts 2*stderr from + # the anchor for the strict floor (95.375), so single-run tails no longer + # trip the outer gate (nvbugs/6506920). The hypothesis-testing threshold + # (~92.3) remains the primary regression gate. - quant_algo: FP8_BLOCK_SCALES spec_dec_algo: DSpark - accuracy: 96.0 + accuracy: 96.475 + stderr: 0.55 Qwen3/Qwen3-4B: - spec_dec_algo: Eagle accuracy: 85.823 diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 6dc4776bb633..16ab39ecc51d 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -181,7 +181,6 @@ full:B300/disaggregated/test_auto_scaling.py::test_service_discovery[http-kv_cac full:B300/disaggregated/test_disaggregated.py::test_disaggregated_ctxpp2_genpp2[TinyLlama-1.1B-Chat-v1.0] SKIP (https://nvbugs/6322073) full:B300/test_e2e.py::test_qwen_e2e_cpprunner_large_new_tokens[DeepSeek-R1-Distill-Qwen-1.5B-DeepSeek-R1-Distill-Qwen-1.5B] SKIP (https://nvbugs/6414760) full:DGX_B200/accuracy/test_disaggregated_serving.py::TestQwen3NextInstruct::test_auto_dtype[use_py_transceiver=True] SKIP (https://nvbugs/6501837) -full:DGX_B200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV4ProDSpark::test_gsm8k_dep8_megamoe_deepgemm SKIP (https://nvbugs/6506920) full:DGX_H100/unittest/_torch/multi_gpu/test_linear.py::test_column_linear[2-unbalanced] SKIP (https://nvbugs/6506918) full:DGX_H200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[disable_skip_indexer] SKIP (https://nvbugs/6476233) full:DGX_H200/accuracy/test_llm_api_pytorch.py::TestDeepSeekV32::test_fp8_blockscale[latency_default] SKIP (https://nvbugs/6476233)