diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6a0deba4ef..2258b96728 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -837,7 +837,8 @@ jobs: # changed `crates//tests/.rs` -> cargo test -p --test # # (plus suite module dirs and `common/` helper dirs — see - # scripts/ci_e2e_scope.py, capped at 12 suites). + # scripts/ci_e2e_scope.py — diff-named suites capped at 12; the perry-codegen + # source map added in #7708 is uncapped and in-process). # # Cost: PRs that touch no integration suite — the large majority — finish in # ~20-30s of checkout + scope computation and never install a toolchain or @@ -860,13 +861,18 @@ jobs: # clock kills legitimately-running suites and we get an uninformative # "cancelled" instead of the per-suite `::error::` — reintroducing, one # level up, exactly what `timeout 1500` below exists to prevent. - # DEFAULT_CAP (ci_e2e_scope.py) = 12 suites - # per-suite bound below = timeout 1500s = 25 min - # 12 x 25 = 300, + 30 min for toolchain/cache/staticlib build = 330 - # Keep these three in sync. This is a backstop, not a budget: each suite is + # DEFAULT_CAP (ci_e2e_scope.py) = 12 diff-named suites + # NAMED_SUITE_TIMEOUT_S = 1500s = 25 min -> 12 x 25 = 300 + # #7708 mapped perry-codegen = 22 suites, uncapped, but in-process: + # MAPPED_SUITE_TIMEOUT_S = 300s = 5 min -> 22 x 5 = 110 + # + 2 known-failure checks at 5 min = 10 + # + 30 min for toolchain/cache/staticlib build + # total = 450 + # Keep these in sync with ci_e2e_scope.py. This is a backstop, not a budget: + # every mapped suite measured 2.2-10.4 s wall clock, each suite is # independently bounded, and the common case selects zero suites and exits # in ~20-30s. - timeout-minutes: 330 + timeout-minutes: 450 env: RUSTC_WRAPPER: sccache SCCACHE_GHA_ENABLED: "false" @@ -891,6 +897,13 @@ jobs: changed_files="$(gh pr view "${{ github.event.pull_request.number }}" \ --json files --jq '.files[].path')" suites="$(printf '%s\n' "$changed_files" | python3 scripts/ci_e2e_scope.py)" + # #7708: the held-out tests, so the run step can skip them AND assert + # each one still fails. + { + echo 'exclusions<> "$GITHUB_OUTPUT" if [ -z "$suites" ]; then echo "No integration suite named by this diff — nothing to run." echo "suites=" >> "$GITHUB_OUTPUT" @@ -945,6 +958,7 @@ jobs: # Bound the heavy per-binary runtime link so the runner doesn't OOM. CARGO_BUILD_JOBS: "1" SUITES: ${{ steps.scope.outputs.suites }} + EXCLUSIONS: ${{ steps.scope.outputs.exclusions }} run: | # `cargo test` never builds the `staticlib` crate-type, so # libperry_{runtime,stdlib}.a don't exist unless built explicitly — @@ -956,17 +970,49 @@ jobs: fi status=0 - while read -r package suite; do + # #7708: every suite runs with its known-failing tests skipped, in + # whichever tier selected it. Holding out a TEST instead of a SUITE is + # what lets `native_proof_regressions` contribute its other 261. + while read -r package suite bound; do [ -n "$package" ] || continue + skips="" + while read -r xpkg xsuite xtest; do + [ -n "$xpkg" ] || continue + if [ "$xpkg" = "$package" ] && [ "$xsuite" = "$suite" ]; then + skips="$skips --skip $xtest" + echo "::notice::skipping known failure $package::$suite::$xtest" + fi + done <<< "$EXCLUSIONS" echo "::group::cargo test -p $package --test $suite" # Per-suite wall-clock bound: a hung compile must not eat the whole - # job budget and hide the other suites' results. - if ! timeout 1500 cargo test -p "$package" --test "$suite"; then + # job budget and hide the other suites' results. Mapped in-process + # suites carry a much tighter bound than a diff-named one that + # shells out to `perry compile` — see ci_e2e_scope.py. + # shellcheck disable=SC2086 + if ! timeout "${bound:-1500}" cargo test -p "$package" --test "$suite" -- $skips; then echo "::error::integration suite failed: $package --test $suite" status=1 fi echo "::endgroup::" done <<< "$SUITES" + + # #7708: the exclusions are self-invalidating. A held-out test that + # now PASSES (or that no longer exists under that name) fails the job, + # so a fix cannot land while leaving its entry behind — the failure + # mode #797 recorded for the parity skip-list. + if printf '%s\n' "$SUITES" | grep -q '^perry-codegen '; then + while read -r xpkg xsuite xtest; do + [ -n "$xpkg" ] || continue + echo "::group::known-failure check $xpkg::$xsuite::$xtest" + out="$(timeout 300 cargo test -p "$xpkg" --test "$xsuite" -- --exact "$xtest" 2>&1 || true)" + printf '%s\n' "$out" + if ! printf '%s\n' "$out" | grep -q '1 failed'; then + echo "::error::$xpkg::$xsuite::$xtest is listed in SUITE_EXCLUSIONS but did not fail (it passed, or no test matched that name). Delete its entry from SUITE_EXCLUSIONS in scripts/ci_e2e_scope.py and let the suite run it." + status=1 + fi + echo "::endgroup::" + done <<< "$EXCLUSIONS" + fi exit "$status" # --------------------------------------------------------------------------- diff --git a/CLAUDE.md b/CLAUDE.md index 960e94bda9..0ac66de295 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Perry is a native TypeScript compiler written in Rust that compiles TypeScript source code directly to native executables. It uses SWC for TypeScript parsing and LLVM for code generation. -**Current Version:** 0.5.1437 +**Current Version:** 0.5.1438 ## TypeScript Parity Status diff --git a/Cargo.lock b/Cargo.lock index 45f0219625..c16fccd1a5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5547,7 +5547,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "perry" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "anyhow", "base64", @@ -5607,14 +5607,14 @@ dependencies = [ [[package]] name = "perry-api-manifest" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "serde", ] [[package]] name = "perry-audio-miniaudio" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "cc", "libc", @@ -5622,7 +5622,7 @@ dependencies = [ [[package]] name = "perry-codegen" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "anyhow", "inkwell", @@ -5639,7 +5639,7 @@ dependencies = [ [[package]] name = "perry-codegen-arkts" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "anyhow", "perry-hir", @@ -5647,7 +5647,7 @@ dependencies = [ [[package]] name = "perry-codegen-glance" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "anyhow", "perry-hir", @@ -5655,7 +5655,7 @@ dependencies = [ [[package]] name = "perry-codegen-js" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "anyhow", "perry-dispatch", @@ -5664,7 +5664,7 @@ dependencies = [ [[package]] name = "perry-codegen-swiftui" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "anyhow", "perry-hir", @@ -5672,7 +5672,7 @@ dependencies = [ [[package]] name = "perry-codegen-wasm" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "anyhow", "base64", @@ -5684,7 +5684,7 @@ dependencies = [ [[package]] name = "perry-codegen-wear-tiles" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "anyhow", "perry-hir", @@ -5692,7 +5692,7 @@ dependencies = [ [[package]] name = "perry-container-compose" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "anyhow", "async-trait", @@ -5721,14 +5721,14 @@ dependencies = [ [[package]] name = "perry-container-e2e" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "anyhow", ] [[package]] name = "perry-diagnostics" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "serde", "serde_json", @@ -5736,7 +5736,7 @@ dependencies = [ [[package]] name = "perry-dispatch" -version = "0.5.1437" +version = "0.5.1438" [[package]] name = "perry-doc-fixture-my-bindings" @@ -5747,7 +5747,7 @@ dependencies = [ [[package]] name = "perry-doc-tests" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "anyhow", "clap", @@ -5762,7 +5762,7 @@ dependencies = [ [[package]] name = "perry-ext-ads" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "block2", "objc2", @@ -5772,7 +5772,7 @@ dependencies = [ [[package]] name = "perry-ext-argon2" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "argon2", "perry-ffi", @@ -5780,7 +5780,7 @@ dependencies = [ [[package]] name = "perry-ext-axios" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "perry-ffi", "reqwest", @@ -5789,7 +5789,7 @@ dependencies = [ [[package]] name = "perry-ext-bcrypt" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "bcrypt", "perry-ffi", @@ -5797,7 +5797,7 @@ dependencies = [ [[package]] name = "perry-ext-better-sqlite3" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "perry-ffi", "rusqlite", @@ -5805,7 +5805,7 @@ dependencies = [ [[package]] name = "perry-ext-cheerio" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "perry-ffi", "scraper", @@ -5813,7 +5813,7 @@ dependencies = [ [[package]] name = "perry-ext-commander" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "perry-ffi", "perry-runtime", @@ -5821,7 +5821,7 @@ dependencies = [ [[package]] name = "perry-ext-cron" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "chrono", "cron", @@ -5831,7 +5831,7 @@ dependencies = [ [[package]] name = "perry-ext-dayjs" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "chrono", "perry-ffi", @@ -5839,7 +5839,7 @@ dependencies = [ [[package]] name = "perry-ext-decimal" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "perry-ffi", "rust_decimal", @@ -5847,7 +5847,7 @@ dependencies = [ [[package]] name = "perry-ext-dotenv" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "perry-ffi", "serde_json", @@ -5855,7 +5855,7 @@ dependencies = [ [[package]] name = "perry-ext-ethers" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "perry-ffi", "rand 0.10.1", @@ -5863,7 +5863,7 @@ dependencies = [ [[package]] name = "perry-ext-events" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "perry-ffi", "perry-runtime", @@ -5871,14 +5871,14 @@ dependencies = [ [[package]] name = "perry-ext-exponential-backoff" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "perry-ffi", ] [[package]] name = "perry-ext-fastify" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "bytes", "http-body-util", @@ -5896,7 +5896,7 @@ dependencies = [ [[package]] name = "perry-ext-fetch" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "bytes", "lazy_static", @@ -5909,7 +5909,7 @@ dependencies = [ [[package]] name = "perry-ext-http" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "bytes", "h2", @@ -5933,7 +5933,7 @@ dependencies = [ [[package]] name = "perry-ext-ioredis" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "lazy_static", "perry-ffi", @@ -5943,7 +5943,7 @@ dependencies = [ [[package]] name = "perry-ext-jsonwebtoken" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "base64", "jsonwebtoken", @@ -5954,7 +5954,7 @@ dependencies = [ [[package]] name = "perry-ext-lru-cache" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "lru", "perry-ffi", @@ -5963,7 +5963,7 @@ dependencies = [ [[package]] name = "perry-ext-moment" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "chrono", "perry-ffi", @@ -5971,7 +5971,7 @@ dependencies = [ [[package]] name = "perry-ext-mongodb" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "bson", "futures-util", @@ -5983,7 +5983,7 @@ dependencies = [ [[package]] name = "perry-ext-mysql2" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "chrono", "perry-ffi", @@ -5993,7 +5993,7 @@ dependencies = [ [[package]] name = "perry-ext-nanoid" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "nanoid", "perry-ffi", @@ -6002,7 +6002,7 @@ dependencies = [ [[package]] name = "perry-ext-net" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "bytes", "perry-ffi", @@ -6015,7 +6015,7 @@ dependencies = [ [[package]] name = "perry-ext-node-forge" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "const-oid 0.9.6", "der 0.7.10", @@ -6034,7 +6034,7 @@ dependencies = [ [[package]] name = "perry-ext-nodemailer" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "lettre", "perry-ffi", @@ -6044,7 +6044,7 @@ dependencies = [ [[package]] name = "perry-ext-pdf" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "perry-ffi", "printpdf", @@ -6052,7 +6052,7 @@ dependencies = [ [[package]] name = "perry-ext-pg" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "perry-ffi", "sqlx", @@ -6061,7 +6061,7 @@ dependencies = [ [[package]] name = "perry-ext-ratelimit" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "governor", "perry-ffi", @@ -6069,7 +6069,7 @@ dependencies = [ [[package]] name = "perry-ext-sharp" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "fast_image_resize", "image", @@ -6079,14 +6079,14 @@ dependencies = [ [[package]] name = "perry-ext-slugify" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "perry-ffi", ] [[package]] name = "perry-ext-streams" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "lazy_static", "perry-ffi", @@ -6095,7 +6095,7 @@ dependencies = [ [[package]] name = "perry-ext-undici" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "perry-ffi", "perry-runtime", @@ -6104,7 +6104,7 @@ dependencies = [ [[package]] name = "perry-ext-uuid" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "perry-ffi", "uuid", @@ -6112,7 +6112,7 @@ dependencies = [ [[package]] name = "perry-ext-validator" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "perry-ffi", "regex", @@ -6122,7 +6122,7 @@ dependencies = [ [[package]] name = "perry-ext-ws" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "futures-util", "lazy_static", @@ -6135,7 +6135,7 @@ dependencies = [ [[package]] name = "perry-ext-zlib" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "brotli", "flate2", @@ -6145,7 +6145,7 @@ dependencies = [ [[package]] name = "perry-ffi" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "dashmap", "once_cell", @@ -6154,7 +6154,7 @@ dependencies = [ [[package]] name = "perry-hir" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "anyhow", "perry-api-manifest", @@ -6172,7 +6172,7 @@ dependencies = [ [[package]] name = "perry-parser" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "anyhow", "perry-diagnostics", @@ -6184,7 +6184,7 @@ dependencies = [ [[package]] name = "perry-runtime" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "anyhow", "base64", @@ -6226,14 +6226,14 @@ dependencies = [ [[package]] name = "perry-runtime-static" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "perry-runtime", ] [[package]] name = "perry-stdlib" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "aes 0.8.4", "aes 0.9.1", @@ -6328,14 +6328,14 @@ dependencies = [ [[package]] name = "perry-stdlib-static" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "perry-stdlib", ] [[package]] name = "perry-transform" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "anyhow", "perry-hir", @@ -6344,14 +6344,14 @@ dependencies = [ [[package]] name = "perry-ui" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "perry-ui-model", ] [[package]] name = "perry-ui-android" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "base64", "itoa", @@ -6368,7 +6368,7 @@ dependencies = [ [[package]] name = "perry-ui-geisterhand" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "rand 0.10.1", "serde", @@ -6378,7 +6378,7 @@ dependencies = [ [[package]] name = "perry-ui-gtk4" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "base64", "cairo-rs 0.22.0", @@ -6401,7 +6401,7 @@ dependencies = [ [[package]] name = "perry-ui-ios" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "base64", "block2", @@ -6417,7 +6417,7 @@ dependencies = [ [[package]] name = "perry-ui-macos" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "base64", "block2", @@ -6432,7 +6432,7 @@ dependencies = [ [[package]] name = "perry-ui-model" -version = "0.5.1437" +version = "0.5.1438" [[package]] name = "perry-ui-test" @@ -6443,11 +6443,11 @@ dependencies = [ [[package]] name = "perry-ui-testkit" -version = "0.5.1437" +version = "0.5.1438" [[package]] name = "perry-ui-tvos" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "base64", "block2", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "perry-ui-visionos" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "base64", "block2", @@ -6479,7 +6479,7 @@ dependencies = [ [[package]] name = "perry-ui-watchos" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "block2", "libc", @@ -6492,7 +6492,7 @@ dependencies = [ [[package]] name = "perry-ui-windows" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "base64", "libc", @@ -6509,14 +6509,14 @@ dependencies = [ [[package]] name = "perry-ui-windows-winui" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "perry-ui-windows", ] [[package]] name = "perry-updater" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "anyhow", "base64", @@ -6532,7 +6532,7 @@ dependencies = [ [[package]] name = "perry-wasm-host" -version = "0.5.1437" +version = "0.5.1438" dependencies = [ "wasmi", ] diff --git a/Cargo.toml b/Cargo.toml index 007baba994..6f879b886c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -315,7 +315,7 @@ codegen-units = 16 codegen-units = 16 [workspace.package] -version = "0.5.1437" +version = "0.5.1438" edition = "2021" license = "MIT" repository = "https://github.com/PerryTS/perry" diff --git a/changelog.d/7748-codegen-suite-coverage.md b/changelog.d/7748-codegen-suite-coverage.md new file mode 100644 index 0000000000..7db37eebce --- /dev/null +++ b/changelog.d/7748-codegen-suite-coverage.md @@ -0,0 +1,13 @@ +### Changed + +- **CI: a `crates/perry-codegen/tests/*.rs` suite can no longer be invisible to per-PR CI (#7708).** Per-PR `cargo-test` runs `--lib --bins`, so no integration suite runs at all; `e2e-scoped` ran only suites the diff *named*, plus a `SOURCE_SUITE_MAP` that mapped all of `crates/perry-codegen/src/` to exactly **three** suites. The crate has **24**. The other 21 were dark, and nothing could say so — a suite in neither the map nor any rule is indistinguishable from one nobody needed to map. + + **Triage of the six failures #7708 counted** (at v0.5.1407): four have since been fixed by unrelated work — `integer_arithmetic_array_push_omits_inbounds_layout_note_and_barrier`, `integer_modulo::i32_counter_mod_unsafe_or_nonliteral_divisors_keep_frem`, `proven_buffer_and_typed_array_reads_are_numeric_operands`, `reassigned_typed_array_store_records_runtime_fallback`. Nothing recorded that either, which is the same bookkeeping failure as #797's parity skip-list. Two remain: `large_object_barriers::large_local_array_push_inbounds_store_emits_precise_slot_barrier`, and `native_proof_regressions::typed_f64_receiver_method_clone_raw_loads_after_composed_guards` (#7506). + + **The map is now complete-by-construction for that crate**: 22 suites, up from 3. `--self-test` (which `e2e-scoped` runs before it installs a toolchain) fails if any suite on disk is neither mapped nor listed in the new `SUITE_EXCLUSIONS`. Adding a suite forces a one-line decision instead of a silent omission. The sabotage case is in the self-test, so the rule is known to be able to fail. + + **Exclusions are per-TEST and self-invalidating in both directions.** Holding out a whole 262-test suite for one failure is how 261 tests' worth of coverage went dark, so the two known failures are skipped by name and the suite still runs. `--self-test` rejects an entry whose suite is gone or which is also mapped; the job then runs *exactly* those tests and fails if one **passes** — so a fix cannot land while leaving its entry behind. Both directions were exercised against a stubbed `cargo` before landing. + + **Cost, measured, one `cargo test -p perry-codegen --test ` per row on arm64 macOS/debug:** every mapped suite completes in 2.2–10.4 s wall clock, nearly all of it cargo's freshness check — the test time is sub-second. Mapped suites are therefore exempt from `--cap` (capping a set the map just completed would reintroduce silent omission one layer up) and carry a 300 s per-suite bound instead of the diff-named 1500 s; the job's `timeout-minutes` math is updated to match. + + Verified before promotion, not after: the whole gate was run with real `cargo` against a built tree — 22 suites green, both known-failure checks still failing as recorded, exit 0. Supersedes #7507, which asked for six of these suites and was blocked on them being green. diff --git a/scripts/ci_e2e_scope.py b/scripts/ci_e2e_scope.py index e695e02afd..68b5ce9bdc 100755 --- a/scripts/ci_e2e_scope.py +++ b/scripts/ci_e2e_scope.py @@ -25,7 +25,12 @@ a `common/` helper module or a `fixtures/` data dir): every suite in that crate can be affected, so all of them are selected. * `SOURCE_SUITE_MAP` — one hand-maintained exception to the rule below, for - the in-process root-lowering suites. See its docstring. + `crates/perry-codegen/src/`, whose suites are in-process compiles rather + than `perry compile` subprocesses. As of #7708 it is COMPLETE for that + crate and checked to stay so: every `crates/perry-codegen/tests/*.rs` must + be either mapped or listed in `SUITE_EXCLUSIONS` with its failing test and + an issue, and `--self-test` fails on one that is neither. See its + docstring. * Everything else selects nothing. In particular a plain `src/` change does NOT map to suites: there is no coverage data to map it with, and a crate-level map (`perry-codegen` -> all 163 `perry` suites) is exactly the @@ -39,7 +44,10 @@ one PR into a multi-hour integration run; the overflow is reported and left to the nightly. +Output is one ` ` line per suite. + Usage: | python3 scripts/ci_e2e_scope.py [--cap N] + python3 scripts/ci_e2e_scope.py --exclusions python3 scripts/ci_e2e_scope.py --self-test """ import os @@ -71,26 +79,115 @@ # the nightly tier by hand — the per-PR `cargo-test` gate is `--lib --bins`, so # it saw none of it. # -# NOT YET LISTED, deliberately: `native_proof_regressions` and -# `native_proof_buffer_views`. Both carry pre-existing failures that are not -# this map's to fix (2 each: typed-array artifact records and integer-modulo / -# typed-f64 clone lowering). A new gate has never been green, and wiring in a -# suite that is red on arrival makes `e2e-scoped` red on most perry-codegen PRs -# — CLAUDE.md hazard 2 with extra steps, since that job is not in branch -# protection and so cannot block anything, and reviewers learn to ignore it. -# Add them here in the same commit that turns them green. +# What changed in #7708: this map listed THREE suites. `crates/perry-codegen/ +# tests/` holds 24, and 23 of them are the same kind of thing — in-process +# compiles of hand-built HIR with `emit_ir_only: true`. The narrowness was not a +# judgment about the other 20; it was what one commit had time to verify. Nothing +# said so, and nothing could: a suite that is in neither the map nor any rule is +# indistinguishable from a suite nobody needed to map. +# +# So the map is now COMPLETE-BY-CONSTRUCTION for this crate. `_assert_map_covers_ +# codegen_suites` (run by `--self-test`, which the `e2e-scoped` job executes +# before it does anything else) requires every `crates/perry-codegen/tests/*.rs` +# on disk to be either mapped here or named in `SUITE_EXCLUSIONS` below. A new +# suite fails CI until someone classifies it. That is the "cannot silently omit +# a suite" property #7708 asked for; the map is still hand-written, but an +# omission is no longer silent. +# +# Measured on arm64 macOS, debug, `--test-threads=1`, one `cargo test -p +# perry-codegen --test ` per row: every one of the 22 mapped suites +# completes in 2.2-10.4 s wall clock, and nearly all of that is cargo's +# freshness check — the test time itself is sub-second (e.g. shadow_slot_hygiene +# 12 tests / 0.10 s, native_proof_buffer_views 36 / 0.2 s). The cost is the +# build, which `e2e-scoped` already pays whenever any perry-codegen suite is in +# scope. They are NOT like the 163 `perry` suites the general refusal protects +# against: no `perry compile` subprocess, no link, no runtime. +# +# Why it exists at all: #7370 changed `crates/perry-codegen/src/` and named no +# suite. `shadow_slot_hygiene` went 0/12 and nothing was red until someone ran +# the nightly tier by hand — the per-PR `cargo-test` gate is `--lib --bins`, so +# it saw none of it. # # Every entry is cross-checked against `tests/` on disk (see `select`): an entry # naming a suite that does not exist FAILS the scope step rather than silently # dropping out, the same rule `scripts/gc_root_dominance_allowlist.json` uses. +_CODEGEN_SRC = "crates/perry-codegen/src/" + +_CODEGEN_SUITES = [ + "app_window_config_options", + "argless_builtin_extra_args", + "class_field_store_pointer_test", + "class_keys_gc_root", + "constructor_recursion", + "destructure_call_location", + "i64_spec_ternary_recursion", + "loop_safepoint_purity", + "macos_bundle_chdir_gate", + "manifest_consistency", + "native_proof_buffer_views", + "node_test_mock_property_presence", + "perry_builtin_name_collision", + "private_guard_declaring_class", + "scalar_replaced_slot_roots", + "shadow_slot_hygiene", + "static_symbol_hygiene", + "temp_root_operand_temporaries", + "typed_feedback", + "typed_shape_declared_at_allocation", + "typed_shape_descriptor", + "typed_shape_descriptors", +] + SOURCE_SUITE_MAP = { - "crates/perry-codegen/src/": [ - ("perry-codegen", "shadow_slot_hygiene"), - ("perry-codegen", "scalar_replaced_slot_roots"), - ("perry-codegen", "temp_root_operand_temporaries"), - ], + _CODEGEN_SRC: [("perry-codegen", suite) for suite in _CODEGEN_SUITES], } +# The suites held OUT of the map, one entry per FAILING TEST, because they are +# red on `main` and a gate that is red on arrival is CLAUDE.md hazard 2 with +# extra steps — `e2e-scoped` is not in branch protection, so a permanently-red +# one teaches reviewers to ignore it. +# +# #7708 counted SIX failures across four suites at v0.5.1407. Four have since +# been fixed by unrelated work and nothing recorded it, which is the same +# bookkeeping failure as #797's parity skip-list: an entry that stops being true +# costs nothing to keep. So these entries are SELF-INVALIDATING in both +# directions: +# +# * the suite must exist on disk (`_assert_exclusions_are_live`), and +# * the named test must still FAIL. `e2e-scoped` runs exactly these tests and +# fails the job if one PASSES, with instructions to delete the entry. A fix +# therefore cannot land while leaving its exclusion behind. +# +# Excluding a TEST rather than a SUITE matters: `native_proof_regressions` is +# 262 tests, and holding all 262 out for one of them is how 261 tests' worth of +# coverage went dark. +SUITE_EXCLUSIONS = [ + ( + "perry-codegen", + "large_object_barriers", + "large_local_array_push_inbounds_store_emits_precise_slot_barrier", + "#7708 — red on main; the other 2 tests in this suite pass.", + ), + ( + "perry-codegen", + "native_proof_regressions", + "typed_f64_receiver_method_clone_raw_loads_after_composed_guards", + "#7506 — the guard-failure edge no longer calls $generic; miscompile or " + "intentional collapse is an open question. The other 261 tests pass.", + ), +] + +# Suites reached through `SOURCE_SUITE_MAP` are exempt from `--cap` and carry a +# tighter per-suite wall-clock bound than a diff-named suite. Both follow from +# the same measured fact: they are in-process and finish in seconds, so neither +# the "a mass rename must not become a multi-hour run" risk the cap exists for +# nor the 25-minute `perry compile` bound applies. Truncating them at the cap +# would reintroduce silent omission one layer up — the mapped set would be +# complete and then quietly cut to 12. +MAPPED_SUITE_TIMEOUT_S = 300 +NAMED_SUITE_TIMEOUT_S = 1500 + + _TESTS_PATH = re.compile(r"^crates/([^/]+)/tests/(.+)$") _PKG_NAME = re.compile(r'^\s*name\s*=\s*"([^"]+)"', re.M) @@ -171,11 +268,28 @@ def _source_map_selection(path: str, root: str): def select(changed, root: str): - """-> sorted list of (package, suite) named by the changed paths.""" + """-> sorted list of (package, suite) named by the changed paths. + + The union of both tiers. `select_split` is the one that keeps them apart; + this stays for callers (and self-test cases) that only care about the set. + """ + mapped, named = select_split(changed, root) + return sorted(set(mapped) | set(named)) + + +def select_split(changed, root: str): + """-> (mapped, named), each a sorted list of (package, suite). + + `mapped` came from `SOURCE_SUITE_MAP`; `named` was named by the diff. They + are kept apart because they are charged differently: only `named` is subject + to `--cap`, and the two carry different per-suite time bounds. A suite in + both is reported as mapped only, so it is never run twice. + """ + from_map = set() selected = set() for path in changed: path = path.strip() - selected |= _source_map_selection(path, root) + from_map |= _source_map_selection(path, root) m = _TESTS_PATH.match(path) if not m: continue @@ -203,7 +317,68 @@ def select(changed, root: str): # the crate can depend on it. for suite in _suites_of(root, crate_dir): selected.add((pkg, suite)) - return sorted(selected) + return sorted(from_map), sorted(selected - from_map) + + +def _assert_map_covers_codegen_suites(root: str) -> None: + """Every `crates/perry-codegen/tests/*.rs` is mapped or explicitly excluded. + + This is the property #7708 asked for. The map stays hand-written — there is + still no coverage data — but a suite nobody classified now FAILS instead of + being indistinguishable from one nobody needed. Adding a suite therefore + forces a one-line decision: map it, or say in `SUITE_EXCLUSIONS` which test + is red and under which issue. + """ + crate_dir = _crate_dir_of(root, "perry-codegen") + if crate_dir is None: + return + on_disk = set(_suites_of(root, crate_dir)) + if not on_disk: + return + mapped = {suite for pkg, suite in SOURCE_SUITE_MAP.get(_CODEGEN_SRC, [])} + excluded = {suite for pkg, suite, _test, _why in SUITE_EXCLUSIONS} + unclassified = sorted(on_disk - mapped - excluded) + if unclassified: + raise SystemExit( + "ci_e2e_scope: these crates/perry-codegen/tests/*.rs suites are in " + "neither SOURCE_SUITE_MAP nor SUITE_EXCLUSIONS: " + + ", ".join(unclassified) + + ". Add each to the map, or to SUITE_EXCLUSIONS with the failing " + "test and an issue number. An unclassified suite is invisible to " + "per-PR CI, which is #7708." + ) + + +def _assert_exclusions_are_live(root: str) -> None: + """An exclusion must name a suite that exists, and must not also be mapped. + + The stale half of the bookkeeping. The other half — "the named test must + still fail" — cannot be answered without running cargo, so `e2e-scoped` + answers it: it runs exactly these tests and fails if one passes. + """ + mapped = {(pkg, suite) for pkg, suite in SOURCE_SUITE_MAP.get(_CODEGEN_SRC, [])} + for pkg, suite, test, why in SUITE_EXCLUSIONS: + if not test or not why: + raise SystemExit( + f"ci_e2e_scope: SUITE_EXCLUSIONS entry {pkg}::{suite} needs both " + "a test name and a reason (issue number)." + ) + if (pkg, suite) in mapped: + raise SystemExit( + f"ci_e2e_scope: {pkg}::{suite} is both mapped and excluded. " + "Pick one — a suite cannot be per-PR coverage and a known " + "failure at the same time." + ) + crate_dir = _crate_dir_of(root, pkg) + on_disk = crate_dir is not None and os.path.isfile( + os.path.join(root, "crates", crate_dir, "tests", suite + ".rs") + ) + if not on_disk: + raise SystemExit( + f"ci_e2e_scope: SUITE_EXCLUSIONS names {pkg}::{suite}, which has " + f"no crates/*/tests/{suite}.rs. Delete the entry in the same " + "commit that renames or deletes the suite (#7708)." + ) def _self_test() -> int: @@ -308,11 +483,69 @@ def touch(rel, body=""): SOURCE_SUITE_MAP.clear() SOURCE_SUITE_MAP.update(saved) + # #7708: the coverage rule must actually be able to fail. Sabotage it with a + # suite on disk that is in neither list and require the raise — a check that + # has never failed is a check nobody has verified (CLAUDE.md, hazard 4). + with tempfile.TemporaryDirectory() as root: + os.makedirs(os.path.join(root, "crates", "perry-codegen", "tests")) + with open( + os.path.join(root, "crates", "perry-codegen", "Cargo.toml"), + "w", + encoding="utf-8", + ) as fh: + fh.write('[package]\nname = "perry-codegen"\n') + for suite in _CODEGEN_SUITES + [s for _p, s, _t, _w in SUITE_EXCLUSIONS]: + open( + os.path.join(root, "crates", "perry-codegen", "tests", suite + ".rs"), + "w", + ).close() + # Classified: passes. + _assert_map_covers_codegen_suites(root) + _assert_exclusions_are_live(root) + # Unclassified: must fail. + open( + os.path.join(root, "crates", "perry-codegen", "tests", "brand_new.rs"), "w" + ).close() + try: + _assert_map_covers_codegen_suites(root) + except SystemExit: + pass + else: + print( + "self-test FAILED: an unclassified perry-codegen suite must fail " + "the scope step (#7708)", + file=sys.stderr, + ) + return 1 + + # The cap applies to diff-named suites only; mapped suites are never cut. + with tempfile.TemporaryDirectory() as root: + os.makedirs(os.path.join(root, "crates", "perry-cc", "tests")) + with open( + os.path.join(root, "crates", "perry-cc", "Cargo.toml"), "w", encoding="utf-8" + ) as fh: + fh.write('[package]\nname = "perry-cc"\n') + for i in range(20): + open( + os.path.join(root, "crates", "perry-cc", "tests", f"s{i:02d}.rs"), "w" + ).close() + named = [f"crates/perry-cc/tests/s{i:02d}.rs" for i in range(20)] + mapped_pairs, named_pairs = select_split(named, root) + if mapped_pairs or len(named_pairs) != 20: + print( + f"self-test FAILED: expected 0 mapped / 20 named, got " + f"{len(mapped_pairs)}/{len(named_pairs)}", + file=sys.stderr, + ) + return 1 + # The real map, against the real repo: every entry must be a suite that # exists here and now, not only in the self-test's fixture tree. real_root = _repo_root() for prefix in SOURCE_SUITE_MAP: select([prefix + "probe.rs"], real_root) + _assert_map_covers_codegen_suites(real_root) + _assert_exclusions_are_live(real_root) print("ci_e2e_scope self-test: ok") return 0 @@ -322,24 +555,38 @@ def main() -> int: if "--self-test" in sys.argv: return _self_test() + root = _repo_root() + + # ` ` for every held-out test, so the runner + # can assert each one still fails and tell the fixer to delete the entry. + if "--exclusions" in sys.argv: + for pkg, suite, test, _why in SUITE_EXCLUSIONS: + print(f"{pkg} {suite} {test}") + return 0 + cap = DEFAULT_CAP if "--cap" in sys.argv: cap = int(sys.argv[sys.argv.index("--cap") + 1]) changed = [line.strip() for line in sys.stdin if line.strip()] - pairs = select(changed, _repo_root()) + mapped, named = select_split(changed, root) - if len(pairs) > cap: + # The cap is a guard against a mass suite rename becoming a multi-hour + # `perry compile` run. Mapped suites are in-process and finish in seconds, + # so capping them would only cut coverage the map just made complete. + if len(named) > cap: print( - f"::notice::{len(pairs)} integration suites named by this diff exceeds the " + f"::notice::{len(named)} integration suites named by this diff exceeds the " f"cap of {cap}; running the first {cap} (sorted). The rest are covered by " f"the nightly full cargo-test.", file=sys.stderr, ) - pairs = pairs[:cap] + named = named[:cap] - for pkg, suite in pairs: - print(f"{pkg} {suite}") + for pkg, suite in mapped: + print(f"{pkg} {suite} {MAPPED_SUITE_TIMEOUT_S}") + for pkg, suite in named: + print(f"{pkg} {suite} {NAMED_SUITE_TIMEOUT_S}") return 0