fix(release): complete hosted binary validation

.github/workflows/release.yml

@@ -58,6 +58,7 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install --yes qemu-user
+          scripts/install-intel-sde.sh "$RUNNER_TEMP/intel-sde"
       - name: Build, execute, package, and validate adaptive x86 CPU bundle
         env:
           EVIDENCE_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
@@ -72,7 +73,8 @@ jobs:
           scripts/build-cpu-release.sh \
             linux-x86_64-glibc-cpu x86_64 stable build-release-cpu-x86 \
             "$(getconf GNU_LIBC_VERSION | awk '{print $2}')" \
-            "$(command -v qemu-x86_64)"
+            "$(command -v qemu-x86_64)" \
+            "$RUNNER_TEMP/intel-sde/sde64"
       - name: Upload exact validated x86 CPU asset triplet
         id: upload
         uses: actions/upload-artifact@v4
@@ -111,6 +113,7 @@ jobs:
           scripts/build-cpu-release.sh \
             linux-aarch64-glibc-cpu aarch64 stable build-release-cpu-arm64 \
             "$(getconf GNU_LIBC_VERSION | awk '{print $2}')" \
+            "$(command -v qemu-aarch64)" \
             "$(command -v qemu-aarch64)"
       - name: Upload exact validated Arm CPU asset triplet
         id: upload
@@ -143,13 +146,15 @@ jobs:
           export SOURCE_DATE_EPOCH
           version=$(sed -n 's/^project(vllm_cpp VERSION \([^ ]*\).*/\1/p' CMakeLists.txt)
           test "$version" = "$VERSION"
+          scripts/install-intel-sde.sh "$RUNNER_TEMP/intel-sde"
           docker run --rm \
             -e EVIDENCE_URL -e SOURCE_DATE_EPOCH -e SOURCE_SHA -e VERSION \
+            -v "$RUNNER_TEMP/intel-sde:/intel-sde:ro" \
             -v "$GITHUB_WORKSPACE:/src" -w /src alpine:3.22 sh -euxc '
-              apk add --no-cache bash binutils build-base cmake file ninja python3 qemu-x86_64
+              apk add --no-cache bash binutils build-base cmake file gcompat ninja python3 qemu-x86_64
               scripts/build-cpu-release.sh \
                 linux-x86_64-musl-cpu-static x86_64 experimental-preview \
-                build-release-cpu-musl 1.2.5 /usr/bin/qemu-x86_64
+                build-release-cpu-musl 1.2.5 /usr/bin/qemu-x86_64 /intel-sde/sde64
             '
           sudo chown -R "$(id -u):$(id -g)" build-release-cpu-musl
       - name: Upload exact validated musl CPU asset triplet

docs/USAGE.md

@@ -192,9 +192,11 @@ bytes; placing either inside those bytes would create a self-reference.
 
 The CPU release helper is the reproducible entry point used by CI. It requires
 an explicit artifact tuple, architecture, channel, build directory, libc ABI,
-and a QEMU userspace emulator. The gate executes every compiled tier under a
-feature-rich CPU model, then executes the baseline and proves rich-tier refusal
-under a feature-poor model before metadata can be generated:
+a feature-poor QEMU userspace emulator, and a feature-rich runner. x86_64 uses
+the SHA256-pinned Intel SDE installed by `scripts/install-intel-sde.sh` so the
+AVX-512 tier is really executed even when the host lacks AVX-512. The gate then
+executes the baseline and proves rich-tier refusal under the feature-poor QEMU
+model before metadata can be generated:
 
 ```sh
 SOURCE_SHA=$(git rev-parse HEAD) \
@@ -203,7 +205,7 @@ SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD) \
 EVIDENCE_URL=https://github.com/mudler/vllm.cpp/actions/runs/EXAMPLE \
 scripts/build-cpu-release.sh \
   linux-x86_64-glibc-cpu x86_64 stable build-release-cpu-x86 \
-  2.39 /usr/bin/qemu-x86_64
+  2.39 /usr/bin/qemu-x86_64 /tmp/intel-sde/sde64
 ```
 
 The corresponding arm64 tuple is `linux-aarch64-glibc-cpu`. The only literal

scripts/build-cpu-release.sh

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-if [[ $# -ne 6 ]]; then
-  echo "usage: $0 ARTIFACT_ID ARCH CHANNEL BUILD_DIR ABI_VERSION POOR_EMULATOR" >&2
+if [[ $# -ne 7 ]]; then
+  echo "usage: $0 ARTIFACT_ID ARCH CHANNEL BUILD_DIR ABI_VERSION POOR_EMULATOR RICH_RUNNER" >&2
   exit 2
 fi
 
@@ -12,6 +12,7 @@ channel=$3
 build_dir=$4
 abi_version=$5
 poor_emulator=$6
+rich_runner=$7
 : "${SOURCE_SHA:?SOURCE_SHA is required}"
 : "${VERSION:?VERSION is required}"
 : "${EVIDENCE_URL:?EVIDENCE_URL is required}"
@@ -51,11 +52,16 @@ metadata_dir="$release_dir/metadata"
 archive="$release_dir/$artifact_id.tar.gz"
 mkdir -p "$release_dir"
 
+rich_runner_kind=qemu
+if [[ "$arch" == x86_64 ]]; then
+  rich_runner_kind=intel-sde
+fi
 python3 scripts/run-cpu-release-gates.py \
   --arch "$arch" \
   --tests-dir "$build_dir/tests" \
   --poor-emulator "$poor_emulator" \
-  --rich-emulator "$poor_emulator" \
+  --rich-runner "$rich_runner" \
+  --rich-runner-kind "$rich_runner_kind" \
   --rich-cpu max \
   --output "$tier_report" \
   --evidence-url "$EVIDENCE_URL"

scripts/build-linux-accelerator-release.sh

@@ -95,6 +95,15 @@ python3 scripts/package-server.py \
   --stage-dir "$stage_dir" \
   --metadata-dir "$metadata_dir" \
   --archive "$archive"
+if [[ "$backend" == cuda ]]; then
+  cuda_stub=$(find /usr/local/cuda -type f -path '*/stubs/libcuda.so' -print -quit)
+  if [[ -z "$cuda_stub" ]]; then
+    echo "CUDA driver stub is required for archive smoke validation" >&2
+    exit 1
+  fi
+  cuda_stub_dir=${cuda_stub%/*}
+  export LD_LIBRARY_PATH="$cuda_stub_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+fi
 python3 scripts/validate-release-archive.py \
   --archive "$archive" \
   --checksum "$archive.sha256" \

scripts/build-macos-release.sh

@@ -18,14 +18,12 @@ mlx_license=${6:-}
 : "${SOURCE_DATE_EPOCH:?SOURCE_DATE_EPOCH is required}"
 
 mlx=OFF
-mlx_metadata_args=()
 if [[ "$artifact_id" == macos-arm64-metal-mlx ]]; then
   mlx=ON
   if [[ -z "$mlx_root" || -z "$mlx_version" || -z "$mlx_license" ]]; then
     echo "MLX artifact requires an exact root, version, and license" >&2
     exit 2
   fi
-  mlx_metadata_args=(--mlx-license "$mlx_license")
 fi
 
 cmake -S . -B "$build_dir" -G Ninja \
@@ -68,7 +66,7 @@ python3 scripts/release_macos_metadata.py \
   --source-clean \
   --abi-version "$(sw_vers -productVersion)" \
   --mlx-version "$mlx_version" \
-  "${mlx_metadata_args[@]}" \
+  --mlx-license "$mlx_license" \
   --compiler "$compiler" \
   --toolchain "$toolchain" \
   --evidence-url "$EVIDENCE_URL"

scripts/install-intel-sde.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -ne 1 ]]; then
+  echo "usage: $0 DESTINATION" >&2
+  exit 2
+fi
+
+destination=$1
+archive_name=sde-external-10.13.1-2026-07-28-lin.tar.xz
+archive_sha256=94e97d623fec54385686e1e7ba65ebc9941748c05ee451423948334892bf2b50
+archive_url=https://downloadmirror.intel.com/924984/$archive_name
+
+if [[ -e "$destination" ]]; then
+  echo "Intel SDE destination already exists: $destination" >&2
+  exit 1
+fi
+
+mkdir -p "$destination"
+archive="$destination/$archive_name"
+curl --fail --location --retry 3 --output "$archive" "$archive_url"
+printf '%s  %s\n' "$archive_sha256" "$archive" | sha256sum --check
+tar --extract --xz --file "$archive" --directory "$destination" --strip-components=1
+rm -f -- "$archive"
+test -x "$destination/sde64"

scripts/run-cpu-release-gates.py

@@ -18,6 +18,9 @@
     "VT_CPU_QUANT_MMLA",
     "VT_CPU_QUANT_REPACK",
 )
+MATMUL_GATE_ARG = (
+    "--test-case=elementwise CPU GEMM: row-strided activation stays bit-exact"
+)
 
 
 class GateError(RuntimeError):
@@ -29,7 +32,8 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--arch", choices=("x86_64", "aarch64"), required=True)
     parser.add_argument("--tests-dir", type=Path, required=True)
     parser.add_argument("--poor-emulator", type=Path, required=True)
-    parser.add_argument("--rich-emulator", type=Path)
+    parser.add_argument("--rich-runner", type=Path)
+    parser.add_argument("--rich-runner-kind", choices=("qemu", "intel-sde"))
     parser.add_argument("--rich-cpu", default="max")
     parser.add_argument("--output", type=Path, required=True)
     parser.add_argument("--evidence-url", required=True)
@@ -43,22 +47,32 @@ def executable(tests_dir: Path, name: str) -> Path:
     return path
 
 
-def display_command(prefix: list[str], env_values: dict[str, str], binary: Path) -> str:
+def test_args(binary: Path, targeted: bool) -> list[str]:
+    return [MATMUL_GATE_ARG] if targeted and binary.name == "test_ops_matmul_elem" else []
+
+
+def display_command(
+    prefix: list[str], env_values: dict[str, str], binary: Path, targeted: bool
+) -> str:
     assignments = " ".join(f"{name}={shlex.quote(value)}" for name, value in env_values.items())
-    command = shlex.join([*prefix, str(binary)])
+    command = shlex.join([*prefix, str(binary), *test_args(binary, targeted)])
     return f"{assignments} {command}".strip()
 
 
 def run_command(
-    prefix: list[str], env_values: dict[str, str], binary: Path, expect_failure: bool = False
+    prefix: list[str],
+    env_values: dict[str, str],
+    binary: Path,
+    expect_failure: bool = False,
+    targeted: bool = False,
 ) -> tuple[str, str]:
-    command = display_command(prefix, env_values, binary)
+    command = display_command(prefix, env_values, binary, targeted)
     environment = os.environ.copy()
     for name in TIER_ENV:
         environment.pop(name, None)
     environment.update(env_values)
     result = subprocess.run(
-        [*prefix, str(binary)],
+        [*prefix, str(binary), *test_args(binary, targeted)],
         env=environment,
         text=True,
         stdout=subprocess.PIPE,
@@ -100,11 +114,16 @@ def run_group(
 def gate(args: argparse.Namespace) -> dict[str, object]:
     tests_dir = args.tests_dir.resolve()
     matmul = executable(tests_dir, "test_ops_matmul_elem")
-    rich_prefix = (
-        [str(args.rich_emulator.resolve()), "-cpu", args.rich_cpu]
-        if args.rich_emulator is not None
-        else []
-    )
+    if (args.rich_runner is None) != (args.rich_runner_kind is None):
+        raise GateError("rich runner and kind must be provided together")
+    rich_prefix: list[str] = []
+    if args.rich_runner is not None:
+        rich_runner = str(args.rich_runner.resolve())
+        rich_prefix = (
+            [rich_runner, "-cpu", args.rich_cpu]
+            if args.rich_runner_kind == "qemu"
+            else [rich_runner, "-skx", "--"]
+        )
     poor_model = "Nehalem" if args.arch == "x86_64" else "cortex-a53"
     poor_prefix = [str(args.poor_emulator.resolve()), "-cpu", poor_model]
     tiers: dict[str, dict[str, str]] = {}
@@ -124,13 +143,15 @@ def gate(args: argparse.Namespace) -> dict[str, object]:
             {"VT_CPU_MATMUL_TIER": "portable"},
             {"VT_CPU_MATMUL_TIER": "sse2"},
         ):
-            command, _ = run_command(poor_prefix, env_values, matmul)
+            command, _ = run_command(poor_prefix, env_values, matmul, targeted=True)
             commands.append(command)
         for env_values in (
             {"VT_CPU_MATMUL_TIER": "sse2+f16c"},
             {"VT_CPU_MATMUL_TIER": "avx2"},
         ):
-            command, _ = run_command(poor_prefix, env_values, matmul, expect_failure=True)
+            command, _ = run_command(
+                poor_prefix, env_values, matmul, expect_failure=True, targeted=True
+            )
             commands.append(command)
         selected_tier = "avx512f"
     else:
@@ -168,15 +189,17 @@ def gate(args: argparse.Namespace) -> dict[str, object]:
             ({"VT_CPU_QUANT_REPACK": "portable"}, quant_repack),
         )
         for env_values, binary in poor_pass:
-            command, _ = run_command(poor_prefix, env_values, binary)
+            command, _ = run_command(poor_prefix, env_values, binary, targeted=True)
             commands.append(command)
         poor_refuse = (
             ({"VT_CPU_Q8_DOT": "sdot"}, quant_dot),
             ({"VT_CPU_QUANT_MMLA": "i8mm"}, quant_dot),
             ({"VT_CPU_QUANT_REPACK": "i8mm"}, quant_repack),
         )
         for env_values, binary in poor_refuse:
-            command, _ = run_command(poor_prefix, env_values, binary, expect_failure=True)
+            command, _ = run_command(
+                poor_prefix, env_values, binary, expect_failure=True, targeted=True
+            )
             commands.append(command)
         selected_tier = "i8mm"
 

tests/scripts/test_cpu_release_gates.py

@@ -34,6 +34,14 @@
 """
 
 
+FAKE_SDE = """#!/bin/sh
+test "$1" = -skx
+test "$2" = --
+shift 2
+exec "$@"
+"""
+
+
 class CpuReleaseGatesContract(unittest.TestCase):
     def fixture(self, scratch: Path) -> tuple[Path, Path]:
         tests = scratch / "tests"
@@ -52,9 +60,15 @@ def fixture(self, scratch: Path) -> tuple[Path, Path]:
         emulator.chmod(0o755)
         return tests, emulator
 
-    def invoke(self, tests: Path, emulator: Path, arch: str, output: Path):
-        return subprocess.run(
-            [
+    def invoke(
+        self,
+        tests: Path,
+        emulator: Path,
+        arch: str,
+        output: Path,
+        rich_runner: Path | None = None,
+    ):
+        argv = [
                 sys.executable,
                 str(TOOL),
                 "--arch",
@@ -67,13 +81,31 @@ def invoke(self, tests: Path, emulator: Path, arch: str, output: Path):
                 str(output),
                 "--evidence-url",
                 "https://github.com/mudler/vllm.cpp/actions/runs/1",
-            ],
+            ]
+        if rich_runner is not None:
+            argv.extend(["--rich-runner", str(rich_runner), "--rich-runner-kind", "intel-sde"])
+        return subprocess.run(
+            argv,
             text=True,
             capture_output=True,
             env={**os.environ, "VLLM_CPP_RELEASE_GATE_TEST": "1"},
             check=False,
         )
 
+    def test_x86_rich_tiers_use_the_explicit_intel_sde_prefix(self) -> None:
+        with tempfile.TemporaryDirectory() as temporary:
+            tests, emulator = self.fixture(Path(temporary))
+            rich_runner = Path(temporary) / "sde64"
+            rich_runner.write_text(FAKE_SDE, encoding="utf-8")
+            rich_runner.chmod(0o755)
+            output = Path(temporary) / "report.json"
+            result = self.invoke(tests, emulator, "x86_64", output, rich_runner)
+            self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+            report = json.loads(output.read_text())
+            self.assertTrue(
+                all("sde64 -skx --" in tier["command"] for tier in report["tiers"].values())
+            )
+
     def test_x86_executes_all_tiers_and_poor_host_refusal(self) -> None:
         with tempfile.TemporaryDirectory() as temporary:
             tests, emulator = self.fixture(Path(temporary))
@@ -88,6 +120,16 @@ def test_x86_executes_all_tiers_and_poor_host_refusal(self) -> None:
             self.assertEqual({row["state"] for row in report["tiers"].values()}, {"passed"})
             self.assertTrue(any("Nehalem" in command for command in report["commands"]))
             self.assertTrue(any("expect-refusal" in command for command in report["commands"]))
+            poor_commands = [command for command in report["commands"] if "Nehalem" in command]
+            self.assertTrue(
+                all("--test-case=elementwise CPU GEMM:" in command for command in poor_commands)
+            )
+            self.assertTrue(
+                all(
+                    "--test-case=elementwise CPU GEMM:" not in tier["command"]
+                    for tier in report["tiers"].values()
+                )
+            )
 
     def test_arm_executes_dotprod_i8mm_and_poor_host_refusal(self) -> None:
         with tempfile.TemporaryDirectory() as temporary:

tests/scripts/test_release_accelerator_metadata.py

@@ -14,6 +14,7 @@
 
 ROOT = Path(__file__).resolve().parents[2]
 TOOL = ROOT / "scripts/release_accelerator_metadata.py"
+BUILD_SCRIPT = ROOT / "scripts/build-linux-accelerator-release.sh"
 SHA = "0123456789abcdef0123456789abcdef01234567"
 SMS = ["80", "86", "87", "89", "90a", "100a", "103a", "110", "120a", "121a"]
 
@@ -110,6 +111,12 @@ def test_cuda_refuses_partial_sm_or_disabled_triton_cache(self) -> None:
                     self.tool.prepare_accelerator_metadata(args)
                 (args.build_dir / "CMakeCache.txt").write_text(cache, encoding="utf-8")
 
+    def test_cuda_archive_smoke_resolves_only_the_external_driver_stub(self) -> None:
+        script = BUILD_SCRIPT.read_text(encoding="utf-8")
+        self.assertIn("*/stubs/libcuda.so", script)
+        self.assertIn("CUDA driver stub is required for archive smoke validation", script)
+        self.assertIn('export LD_LIBRARY_PATH="$cuda_stub_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"', script)
+
 
 if __name__ == "__main__":
     unittest.main()