From 48eb5c7323fcfacae24c88efa1a7361c429556a0 Mon Sep 17 00:00:00 2001 From: zackees Date: Sun, 21 Jun 2026 23:06:16 -0700 Subject: [PATCH] perf(install): mtime-skip fast path + structured artifact discovery (#2 items 6+8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two upgrades to `ci/build_wheel.py` ported from FastLED/fbuild#743. 1. mtime-skip fast path. `_staged_binary_is_up_to_date()` walks the workspace's cargo inputs (Cargo.toml, Cargo.lock, rust-toolchain.toml, crates/**/Cargo.toml, crates/**/*.rs) and compares each st_mtime against the staged `src/template_python_rust_cmd/_bin/template-cli`. If staged is newer, skip cargo entirely — `_ensure_staged_cli_binary()` returns the already-staged file. Triggered on no-op reinstalls (version bumps, lockfile churn, --reinstall-package). Even cargo's "Fresh" pass walks the workspace and burns wall-clock seconds; an mtime check is milliseconds. fbuild measured 14.9s → 1.1s on its forced reinstall path. 2. Structured artifact discovery. `build_cli_binary()` now invokes cargo with `--message-format=json-render-diagnostics` and walks the JSON artifact stream for `reason == "compiler-artifact"` + `target.name == "template-cli"` + non-null `executable`. Fallback: `_find_cli_executable_by_search()` probes the target root (respecting CARGO_TARGET_DIR) and every per-host-triple subdir for cases where cargo emits no compiler-artifact line (fully cached `Fresh` runs) or where the build is configured with a host triple (CARGO_BUILD_TARGET). Replaces the previous hardcoded `target/release/template-cli` path, which would have masked the pinned CARGO_TARGET_DIR landed in the previous PR. Refs zackees/template-python-rust-cmd#2 (items 6 + 8). Co-Authored-By: Claude Opus 4.7 --- ci/build_wheel.py | 146 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 137 insertions(+), 9 deletions(-) diff --git a/ci/build_wheel.py b/ci/build_wheel.py index 4a056e6..ca070c9 100644 --- a/ci/build_wheel.py +++ b/ci/build_wheel.py @@ -4,6 +4,7 @@ # /// from __future__ import annotations +import json import os import shutil import platform @@ -17,6 +18,7 @@ CLI_TARGET_NAME = ( "template-cli.exe" if platform.system() == "Windows" else "template-cli" ) +STAGED_BINARY_PATH = PACKAGE_BIN_DIR / CLI_TARGET_NAME # Pin cargo's target directory to a stable home-dir path so PEP 517 # isolated builds reuse cargo's incremental fingerprint cache across @@ -46,20 +48,146 @@ def _cargo_target_root() -> Path: return ROOT / "target" +def _iter_cargo_inputs() -> list[Path]: + """Files that, if newer than the staged binary, invalidate the cache.""" + patterns = ( + "Cargo.toml", + "Cargo.lock", + "rust-toolchain.toml", + "crates/**/Cargo.toml", + "crates/**/*.rs", + ) + paths: list[Path] = [] + for pat in patterns: + paths.extend(ROOT.glob(pat)) + return paths + + +def _staged_binary_is_up_to_date() -> bool: + """True if the staged binary exists and is newer than every cargo input. + + Skips the cargo invocation entirely on no-op reinstalls (version + bumps, lockfile churn, --reinstall-package). Even cargo's "Fresh" + pass walks the workspace and burns wall-clock seconds; an mtime + check is milliseconds. See FastLED/fbuild#743 and + zackees/template-python-rust-cmd#2 (item 6). + """ + if not STAGED_BINARY_PATH.is_file(): + return False + staged_mtime = STAGED_BINARY_PATH.stat().st_mtime + for path in _iter_cargo_inputs(): + try: + if path.stat().st_mtime > staged_mtime: + return False + except FileNotFoundError: + # Glob race — treat as changed and rebuild. + return False + return True + + +def _find_cli_executable_from_json(stdout: str) -> Path | None: + """Walk cargo's structured artifact stream for the `template-cli` exec. + + cargo emits one JSON object per line; the artifact we want has + `reason == "compiler-artifact"`, `target.name == "template-cli"`, + and a non-null `executable` field. We keep the *last* match because + cargo emits one artifact per crate target kind and the bin artifact + is what we want (matches `cargo install`'s selection rule). + """ + found: Path | None = None + for line in stdout.splitlines(): + line = line.strip() + if not line or not line.startswith("{"): + continue + try: + msg = json.loads(line) + except json.JSONDecodeError: + continue + if msg.get("reason") != "compiler-artifact": + continue + target = msg.get("target") or {} + if target.get("name") != "template-cli": + continue + executable = msg.get("executable") + if executable: + found = Path(executable) + return found + + +def _find_cli_executable_by_search() -> Path | None: + """Fallback when cargo emits no compiler-artifact line for the binary. + + Cargo skips the `compiler-artifact` JSON line for fully-cached + builds (everything `Fresh`), so the primary discovery path returns + None there. Probe `target/release/` and every per-host-triple + subdir of `target/`. See zackees/template-python-rust-cmd#2 (item 8). + """ + target_root = _cargo_target_root() + candidates = [target_root / "release" / CLI_TARGET_NAME] + if target_root.is_dir(): + for child in target_root.iterdir(): + candidate = child / "release" / CLI_TARGET_NAME + if candidate.is_file(): + candidates.append(candidate) + for candidate in candidates: + if candidate.is_file(): + return candidate + return None + + def build_cli_binary() -> Path: - if run(["cargo", "build", "--release", "-p", "template-cli"]) != 0: - raise SystemExit(1) - binary = _cargo_target_root() / "release" / CLI_TARGET_NAME - if not binary.exists(): - raise SystemExit(f"expected native CLI binary at {binary}") + cmd = [ + "cargo", + "build", + "--release", + "-p", + "template-cli", + "--message-format=json-render-diagnostics", + ] + # stderr passes through; stdout is captured for the artifact stream. + proc = subprocess.run( + cmd, + cwd=ROOT, + stdout=subprocess.PIPE, + stderr=None, + check=False, + text=True, + encoding="utf-8", + ) + if proc.returncode != 0: + raise SystemExit(proc.returncode) + binary = _find_cli_executable_from_json(proc.stdout) + if binary is None or not binary.is_file(): + binary = _find_cli_executable_by_search() + if binary is None or not binary.is_file(): + raise SystemExit( + "cargo build succeeded but no `template-cli` binary was found.\n" + "Searched cargo's JSON artifact stream and " + f"{_cargo_target_root()}/release/{CLI_TARGET_NAME} (plus per-target subdirs)." + ) return binary def stage_cli_binary(binary: Path) -> Path: PACKAGE_BIN_DIR.mkdir(parents=True, exist_ok=True) - staged = PACKAGE_BIN_DIR / CLI_TARGET_NAME - shutil.copy2(binary, staged) - return staged + shutil.copy2(binary, STAGED_BINARY_PATH) + return STAGED_BINARY_PATH + + +def _ensure_staged_cli_binary() -> Path: + """Stage `template-cli` into the package, skipping cargo if cached. + + Fast path: if the staged binary is newer than every cargo input, the + file on disk already reflects the current source — return it without + touching cargo. Slow path: build via cargo and copy. + """ + if _staged_binary_is_up_to_date(): + print( + f" staged binary up-to-date ({STAGED_BINARY_PATH}); skipping cargo", + file=sys.stderr, + ) + return STAGED_BINARY_PATH + return stage_cli_binary(build_cli_binary()) def remove_staged_binary(staged: Path) -> None: @@ -114,7 +242,7 @@ def verify_artifacts() -> int: def main() -> int: - staged = stage_cli_binary(build_cli_binary()) + staged = _ensure_staged_cli_binary() try: if build_python_artifacts() != 0: return 1