diff --git a/.github/workflows/gc-parse-churn-gate.yml b/.github/workflows/gc-parse-churn-gate.yml new file mode 100644 index 0000000000..03ab287d0f --- /dev/null +++ b/.github/workflows/gc-parse-churn-gate.yml @@ -0,0 +1,197 @@ +name: GC Parse-Churn Layout Gate + +# Promotes the "tape=0 + from-space-scan parse-then-churn" check to CI (#7647). +# +# WHY THIS EXISTS +# +# #7643 measured that `PERRY_JSON_TAPE=0` + `PERRY_GC_FROMSPACE_SCAN=1` over a +# parse-then-churn workload is a known-good end-to-end detector for the whole +# layout-state family (#7630 / #7633 / #7635 / #7644): with the JSON +# materialiser's finalize sabotaged to always claim `POINTER_FREE`, it reports +# `dangling=8000 owners=4000` and the binary SIGBUSes; clean, `dangling=0` and +# exit 0. #7643/#7644 shipped workload-free unit tests for two invariants that +# cannot be defeated by GC timing or a lazy path -- the right primary guard -- +# but neither can catch a NEW materialiser path that forgets to finalize at +# all, since a hand-built unit-test object never exercises a real call site. +# That is what this job is for. Nothing ran the end-to-end check in CI before +# this file; #7647 is that promotion. +# +# THIS JOB IS DESIGNED TO BE ABLE TO FAIL, checked against CLAUDE.md's "four +# ways a gate can be unable to fail": +# +# 1. no `continue-on-error`, no `|| true`, no pipe between the gate script +# and the shell's exit status. `scripts/gc_parse_churn_layout_gate.sh` +# itself runs under `set -euo pipefail` and its final line is a bare +# `exit "$CHECK_RC"`. +# 2. NOT wired into branch protection's required contexts by this change, +# deliberately: a new gate has never been green, so promoting it +# immediately would block every open PR (CLAUDE.md's corollary). That is +# a maintainer action, to be taken after this job's first observed green +# run on `main` -- and per the corollary, taking it is not optional +# follow-through: `gc-root-dominance` sat red on `main` for weeks after +# the same deferral because the second step was never done. +# 3. `concurrency` below cancels `pull_request` runs only; `push` (main) +# runs are keyed on the commit SHA so they queue instead of cancelling +# each other (the `gc-ratchet` regression this guards against is #7205). +# 4. the subject must be LIVE, not merely quiet. +# `scripts/gc_parse_churn_layout_check.py` -- the actual verdict logic, +# not this workflow -- rejects a run whose from-space scan never even +# ran, one where zero copying minors relocated anything (the moving +# collector is opt-in: `PERRY_GC_MOVING_LOOP_POLLS=1`, both at compile +# time and run time, is the ONLY configuration that exercises it end to +# end per #7161's stopgap), and one whose from-space census stayed too +# small to have eagerly materialised the record cohort (the #7635 +# vacuity: a lazily-parsed cohort leaves only a handful of live objects, +# nowhere near the fixture's record count, so a clean scan over it would +# mean nothing). `scripts/gc_parse_churn_layout_check.py --self-test` +# proves the checker itself can say no on all of the above, in both +# directions, and runs on every PR regardless of relevance (it needs no +# compiler). The end-to-end sabotage run -- the JSON materialiser's +# finalize forced to always claim `POINTER_FREE`, #7635's exact +# mutation -- is documented in this gate's introducing PR rather than +# re-run on every CI invocation, matching how `gc-root-dominance` and +# `gc-moving-witnesses` treat their own one-time proof. +# +# WHAT IT DOES +# +# One script, no arms to choose between: +# +# scripts/gc_parse_churn_layout_gate.sh +# +# which compiles `scripts/fixtures/gc_parse_churn_layout_state.ts` with +# `PERRY_GC_MOVING_LOOP_POLLS=1` (a compile-time gate as well as a runtime +# one -- see that script's own comments), runs it under +# `PERRY_JSON_TAPE=0 PERRY_GC_FROMSPACE_SCAN_ABORT=1 PERRY_GC_DIAG=1`, and +# hands the captured stdout/stderr/exit-code to the Python checker above. +# +# WHAT IT DELIBERATELY DOES NOT COVER +# +# This is a single, deliberately narrow probe (4,000 JSON records, 2 +# pointer-bearing string fields each, matching #7643's own reproduction +# shape) -- not a sweep over every layout-state shape the codebase can +# produce. It is the end-to-end complement to #7643/#7644's unit tests, not a +# replacement for them, and not a substitute for `gc-moving-witnesses` (the +# #7154 stale-root reproducer family) or `gc-root-dominance` (the static +# root-store-dominance check), which cover different hazard classes. + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # One group per push COMMIT so `main` runs queue instead of cancelling each + # other (a `cancel-in-progress: true` scoped to the whole group would still + # cancel a pending `main` run the moment a new one enters the SAME group -- + # #7205, measured on gc-ratchet with three consecutive `main` runs + # cancelled and zero executed). PR runs are cancelled on superseding pushes. + group: gc-parse-churn-gate-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CARGO_TERM_COLOR: always + +jobs: + self-test-checker: + # Cheap, compiler-free, and runs on every PR regardless of relevance: the + # checker logic must always be able to say no. See CLAUDE.md's GC knob + # kill-policy and the four-ways-a-gate-cannot-fail rule this whole job + # exists to satisfy point 4 of. + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Self-test the verdict logic + run: python3 scripts/gc_parse_churn_layout_check.py --self-test + + gc-parse-churn-gate: + runs-on: ubuntu-latest + # Matches gc-moving-witnesses/gc-stress: the witness run itself is + # minutes, but a cold cargo cache makes the compiler build the whole cost + # of this job. + timeout-minutes: 90 + permissions: + contents: read + pull-requests: read + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Decide whether this change can affect the layout-state / parse-churn path + id: relevance + env: + GH_TOKEN: ${{ github.token }} + EVENT_NAME: ${{ github.event_name }} + REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + set -euo pipefail + if [[ "$EVENT_NAME" != "pull_request" ]]; then + echo "run=true" >> "$GITHUB_OUTPUT" + echo "Not a pull request; running the gate." + exit 0 + fi + gh api "repos/$REPOSITORY/pulls/$PR_NUMBER/files" --paginate --jq '.[].filename' > changed.txt + # Deliberately broad, same rationale as gc-moving-witnesses.yml: + # anything under crates/ can move the collector or change how JSON + # parses, and the filter exists only to spare docs-only PRs a + # compiler build. + if grep -qE '^(crates/|scripts/gc_parse_churn_layout_(gate\.sh|check\.py)$|scripts/fixtures/gc_parse_churn_layout_state\.ts$|Cargo\.(toml|lock)$|\.github/workflows/gc-parse-churn-gate\.yml$)' changed.txt; then + echo "run=true" >> "$GITHUB_OUTPUT" + echo "Change touches a collector- or JSON-parse-relevant path; running the gate." + else + echo "run=false" >> "$GITHUB_OUTPUT" + echo "No relevant paths changed." + fi + + - name: Install Rust toolchain + if: steps.relevance.outputs.run == 'true' + uses: dtolnay/rust-toolchain@stable + - uses: ./.github/actions/setup-llvm22 + if: steps.relevance.outputs.run == 'true' + + - uses: Swatinem/rust-cache@v2 + if: steps.relevance.outputs.run == 'true' + with: + # Shared with gc-stress/gc-moving-witnesses (same package set), so + # this job usually starts from an already-warm cache. + shared-key: "${{ runner.os }}-perry" + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Install clang + if: steps.relevance.outputs.run == 'true' + run: | + sudo apt-get update + sudo apt-get install -y clang + + - name: Build perry and the runtime archives + if: steps.relevance.outputs.run == 'true' + env: + # Matches cargo-test/gc-stress/gc-moving-witnesses: works around an + # lld SIGBUS on this shared runner during large links. + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS: "-C linker-features=-lld" + run: | + set -euo pipefail + # perry-runtime/perry-stdlib are rlib-only; libperry_{runtime,stdlib}.a + # come from the *-static wrapper crates. Building without them links + # a stale archive and makes this whole gate vacuous in a way nothing + # downstream can detect (CLAUDE.md's "Verifying a runtime change"). + cargo build --release \ + -p perry -p perry-runtime -p perry-stdlib \ + -p perry-runtime-static -p perry-stdlib-static + for artifact in perry libperry_runtime.a libperry_stdlib.a; do + test -s "target/release/$artifact" \ + || { echo "::error::target/release/$artifact was not produced"; exit 1; } + done + + # GATING. No pipe, no `|| true`: this step's exit status is the gate. + - name: Run the parse-then-churn layout-state gate + if: steps.relevance.outputs.run == 'true' + run: scripts/gc_parse_churn_layout_gate.sh target/release/perry diff --git a/CLAUDE.md b/CLAUDE.md index 39af176637..e9c1f3b71f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Perry is a native TypeScript compiler written in Rust that compiles TypeScript source code directly to native executables. It uses SWC for TypeScript parsing and LLVM for code generation. -**Current Version:** 0.5.1411 +**Current Version:** 0.5.1412 ## TypeScript Parity Status diff --git a/Cargo.lock b/Cargo.lock index 603bbe06b9..15c646074f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5547,7 +5547,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "perry" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "anyhow", "base64", @@ -5607,14 +5607,14 @@ dependencies = [ [[package]] name = "perry-api-manifest" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "serde", ] [[package]] name = "perry-audio-miniaudio" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "cc", "libc", @@ -5622,7 +5622,7 @@ dependencies = [ [[package]] name = "perry-codegen" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "anyhow", "inkwell", @@ -5639,7 +5639,7 @@ dependencies = [ [[package]] name = "perry-codegen-arkts" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "anyhow", "perry-hir", @@ -5647,7 +5647,7 @@ dependencies = [ [[package]] name = "perry-codegen-glance" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "anyhow", "perry-hir", @@ -5655,7 +5655,7 @@ dependencies = [ [[package]] name = "perry-codegen-js" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "anyhow", "perry-dispatch", @@ -5664,7 +5664,7 @@ dependencies = [ [[package]] name = "perry-codegen-swiftui" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "anyhow", "perry-hir", @@ -5672,7 +5672,7 @@ dependencies = [ [[package]] name = "perry-codegen-wasm" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "anyhow", "base64", @@ -5684,7 +5684,7 @@ dependencies = [ [[package]] name = "perry-codegen-wear-tiles" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "anyhow", "perry-hir", @@ -5692,7 +5692,7 @@ dependencies = [ [[package]] name = "perry-container-compose" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "anyhow", "async-trait", @@ -5721,14 +5721,14 @@ dependencies = [ [[package]] name = "perry-container-e2e" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "anyhow", ] [[package]] name = "perry-diagnostics" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "serde", "serde_json", @@ -5736,7 +5736,7 @@ dependencies = [ [[package]] name = "perry-dispatch" -version = "0.5.1411" +version = "0.5.1412" [[package]] name = "perry-doc-fixture-my-bindings" @@ -5747,7 +5747,7 @@ dependencies = [ [[package]] name = "perry-doc-tests" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "anyhow", "clap", @@ -5762,7 +5762,7 @@ dependencies = [ [[package]] name = "perry-ext-ads" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "block2", "objc2", @@ -5772,7 +5772,7 @@ dependencies = [ [[package]] name = "perry-ext-argon2" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "argon2", "perry-ffi", @@ -5780,7 +5780,7 @@ dependencies = [ [[package]] name = "perry-ext-axios" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "perry-ffi", "reqwest", @@ -5789,7 +5789,7 @@ dependencies = [ [[package]] name = "perry-ext-bcrypt" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "bcrypt", "perry-ffi", @@ -5797,7 +5797,7 @@ dependencies = [ [[package]] name = "perry-ext-better-sqlite3" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "perry-ffi", "rusqlite", @@ -5805,7 +5805,7 @@ dependencies = [ [[package]] name = "perry-ext-cheerio" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "perry-ffi", "scraper", @@ -5813,7 +5813,7 @@ dependencies = [ [[package]] name = "perry-ext-commander" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "perry-ffi", "perry-runtime", @@ -5821,7 +5821,7 @@ dependencies = [ [[package]] name = "perry-ext-cron" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "chrono", "cron", @@ -5831,7 +5831,7 @@ dependencies = [ [[package]] name = "perry-ext-dayjs" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "chrono", "perry-ffi", @@ -5839,7 +5839,7 @@ dependencies = [ [[package]] name = "perry-ext-decimal" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "perry-ffi", "rust_decimal", @@ -5847,7 +5847,7 @@ dependencies = [ [[package]] name = "perry-ext-dotenv" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "perry-ffi", "serde_json", @@ -5855,7 +5855,7 @@ dependencies = [ [[package]] name = "perry-ext-ethers" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "perry-ffi", "rand 0.10.1", @@ -5863,7 +5863,7 @@ dependencies = [ [[package]] name = "perry-ext-events" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "perry-ffi", "perry-runtime", @@ -5871,14 +5871,14 @@ dependencies = [ [[package]] name = "perry-ext-exponential-backoff" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "perry-ffi", ] [[package]] name = "perry-ext-fastify" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "bytes", "http-body-util", @@ -5896,7 +5896,7 @@ dependencies = [ [[package]] name = "perry-ext-fetch" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "bytes", "lazy_static", @@ -5909,7 +5909,7 @@ dependencies = [ [[package]] name = "perry-ext-http" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "bytes", "h2", @@ -5933,7 +5933,7 @@ dependencies = [ [[package]] name = "perry-ext-ioredis" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "lazy_static", "perry-ffi", @@ -5943,7 +5943,7 @@ dependencies = [ [[package]] name = "perry-ext-jsonwebtoken" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "base64", "jsonwebtoken", @@ -5954,7 +5954,7 @@ dependencies = [ [[package]] name = "perry-ext-lru-cache" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "lru", "perry-ffi", @@ -5963,7 +5963,7 @@ dependencies = [ [[package]] name = "perry-ext-moment" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "chrono", "perry-ffi", @@ -5971,7 +5971,7 @@ dependencies = [ [[package]] name = "perry-ext-mongodb" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "bson", "futures-util", @@ -5983,7 +5983,7 @@ dependencies = [ [[package]] name = "perry-ext-mysql2" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "chrono", "perry-ffi", @@ -5993,7 +5993,7 @@ dependencies = [ [[package]] name = "perry-ext-nanoid" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "nanoid", "perry-ffi", @@ -6002,7 +6002,7 @@ dependencies = [ [[package]] name = "perry-ext-net" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "bytes", "perry-ffi", @@ -6015,7 +6015,7 @@ dependencies = [ [[package]] name = "perry-ext-node-forge" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "const-oid 0.9.6", "der 0.7.10", @@ -6034,7 +6034,7 @@ dependencies = [ [[package]] name = "perry-ext-nodemailer" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "lettre", "perry-ffi", @@ -6044,7 +6044,7 @@ dependencies = [ [[package]] name = "perry-ext-pdf" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "perry-ffi", "printpdf", @@ -6052,7 +6052,7 @@ dependencies = [ [[package]] name = "perry-ext-pg" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "perry-ffi", "sqlx", @@ -6061,7 +6061,7 @@ dependencies = [ [[package]] name = "perry-ext-ratelimit" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "governor", "perry-ffi", @@ -6069,7 +6069,7 @@ dependencies = [ [[package]] name = "perry-ext-sharp" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "fast_image_resize", "image", @@ -6079,14 +6079,14 @@ dependencies = [ [[package]] name = "perry-ext-slugify" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "perry-ffi", ] [[package]] name = "perry-ext-streams" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "lazy_static", "perry-ffi", @@ -6095,7 +6095,7 @@ dependencies = [ [[package]] name = "perry-ext-undici" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "perry-ffi", "perry-runtime", @@ -6104,7 +6104,7 @@ dependencies = [ [[package]] name = "perry-ext-uuid" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "perry-ffi", "uuid", @@ -6112,7 +6112,7 @@ dependencies = [ [[package]] name = "perry-ext-validator" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "perry-ffi", "regex", @@ -6122,7 +6122,7 @@ dependencies = [ [[package]] name = "perry-ext-ws" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "futures-util", "lazy_static", @@ -6135,7 +6135,7 @@ dependencies = [ [[package]] name = "perry-ext-zlib" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "brotli", "flate2", @@ -6145,7 +6145,7 @@ dependencies = [ [[package]] name = "perry-ffi" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "dashmap", "once_cell", @@ -6154,7 +6154,7 @@ dependencies = [ [[package]] name = "perry-hir" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "anyhow", "perry-api-manifest", @@ -6172,7 +6172,7 @@ dependencies = [ [[package]] name = "perry-parser" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "anyhow", "perry-diagnostics", @@ -6184,7 +6184,7 @@ dependencies = [ [[package]] name = "perry-runtime" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "anyhow", "base64", @@ -6226,14 +6226,14 @@ dependencies = [ [[package]] name = "perry-runtime-static" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "perry-runtime", ] [[package]] name = "perry-stdlib" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "aes 0.8.4", "aes 0.9.1", @@ -6328,14 +6328,14 @@ dependencies = [ [[package]] name = "perry-stdlib-static" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "perry-stdlib", ] [[package]] name = "perry-transform" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "anyhow", "perry-hir", @@ -6344,14 +6344,14 @@ dependencies = [ [[package]] name = "perry-ui" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "perry-ui-model", ] [[package]] name = "perry-ui-android" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "base64", "itoa", @@ -6368,7 +6368,7 @@ dependencies = [ [[package]] name = "perry-ui-geisterhand" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "rand 0.10.1", "serde", @@ -6378,7 +6378,7 @@ dependencies = [ [[package]] name = "perry-ui-gtk4" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "base64", "cairo-rs 0.22.0", @@ -6401,7 +6401,7 @@ dependencies = [ [[package]] name = "perry-ui-ios" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "base64", "block2", @@ -6417,7 +6417,7 @@ dependencies = [ [[package]] name = "perry-ui-macos" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "base64", "block2", @@ -6432,7 +6432,7 @@ dependencies = [ [[package]] name = "perry-ui-model" -version = "0.5.1411" +version = "0.5.1412" [[package]] name = "perry-ui-test" @@ -6443,11 +6443,11 @@ dependencies = [ [[package]] name = "perry-ui-testkit" -version = "0.5.1411" +version = "0.5.1412" [[package]] name = "perry-ui-tvos" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "base64", "block2", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "perry-ui-visionos" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "base64", "block2", @@ -6479,7 +6479,7 @@ dependencies = [ [[package]] name = "perry-ui-watchos" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "block2", "libc", @@ -6492,7 +6492,7 @@ dependencies = [ [[package]] name = "perry-ui-windows" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "base64", "libc", @@ -6509,14 +6509,14 @@ dependencies = [ [[package]] name = "perry-ui-windows-winui" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "perry-ui-windows", ] [[package]] name = "perry-updater" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "anyhow", "base64", @@ -6532,7 +6532,7 @@ dependencies = [ [[package]] name = "perry-wasm-host" -version = "0.5.1411" +version = "0.5.1412" dependencies = [ "wasmi", ] diff --git a/Cargo.toml b/Cargo.toml index 3c3c0df9ba..fcc328f707 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -315,7 +315,7 @@ codegen-units = 16 codegen-units = 16 [workspace.package] -version = "0.5.1411" +version = "0.5.1412" edition = "2021" license = "MIT" repository = "https://github.com/PerryTS/perry" diff --git a/changelog.d/7711-parse-churn-layout-gate.md b/changelog.d/7711-parse-churn-layout-gate.md new file mode 100644 index 0000000000..3e4666dc27 --- /dev/null +++ b/changelog.d/7711-parse-churn-layout-gate.md @@ -0,0 +1,22 @@ +**gate(gc): promote the parse-then-churn layout-state check to CI (#7647)** + +`PERRY_JSON_TAPE=0` + `PERRY_GC_FROMSPACE_SCAN=1` over a parse-then-churn +workload — the known-good end-to-end detector for the layout-state family +(#7630/#7633/#7635/#7643/#7644) that #7643 measured but nothing ran in +CI — now runs as `gc-parse-churn-gate.yml` on every PR and `main` push +(`scripts/gc_parse_churn_layout_gate.sh`). The verdict +(`scripts/gc_parse_churn_layout_check.py`) requires correctness, liveness (a +copying minor actually relocated objects), and eagerness (the from-space +scan's own census reached the record count, so the cohort was not still on +the lazy tape) before it can pass — `--self-test` proves it can say no on +each axis. Not yet required in branch protection; that is a maintainer +action for after the first green run on `main`. + +Along the way, fixed a real false-positive source in +`PERRY_GC_FROMSPACE_SCAN`: string allocations left their 0-7 byte +alignment padding uninitialized, so leftover arena bytes there could +occasionally decode as a plausible pointer. Harmless to every string API +(all bounded by `byte_len`, never `GcHeader.size`) but not to a scan that +trusts `GcHeader.size` as the payload's true extent. Fixed at +`string_storage_alloc`'s single choke point, mirroring the existing +`TAG_HOLE` fix for array-growth slack. diff --git a/crates/perry-runtime/src/string/mod.rs b/crates/perry-runtime/src/string/mod.rs index 308c5f62b9..6df9e22ea0 100644 --- a/crates/perry-runtime/src/string/mod.rs +++ b/crates/perry-runtime/src/string/mod.rs @@ -504,6 +504,7 @@ pub(crate) fn string_storage_alloc(capacity: u32) -> (*mut StringHeader, *mut u8 let raw = crate::arena::arena_alloc_gc(payload_size, 8, crate::gc::GC_TYPE_STRING); let ptr = raw as *mut StringHeader; let data = unsafe { raw.add(std::mem::size_of::()) }; + zero_alignment_padding_tail(raw, payload_size); (ptr, data) } @@ -513,9 +514,59 @@ pub(crate) fn string_storage_alloc_longlived(capacity: u32) -> (*mut StringHeade let raw = crate::arena::arena_alloc_gc_longlived(payload_size, 8, crate::gc::GC_TYPE_STRING); let ptr = raw as *mut StringHeader; let data = unsafe { raw.add(std::mem::size_of::()) }; + zero_alignment_padding_tail(raw, payload_size); (ptr, data) } +/// #7647: `arena_alloc_gc`/`arena_alloc_gc_longlived`/`arena_alloc_gc_old` all +/// round a request's total size UP to 8-byte alignment +/// (`gc_padded_total_size` in `arena/allocators.rs`), so a payload whose own +/// natural size is not already a multiple of 8 gets up to 7 trailing bytes +/// that are part of the allocation (`GcHeader.size`, what the collector and +/// every heap-walking pass treat as this object's true extent) but were +/// never requested by, or written by, the caller. +/// +/// For every other `GC_TYPE_*` this trailing pad is a non-issue in practice +/// because the type's own construction writes every declared field (an +/// Object/Closure/Array literal has no "unstated" slot) -- and where it +/// legitimately can, it is already handled: `js_array_grow`'s +/// `[old_capacity, new_capacity)` slack is explicitly `TAG_HOLE`-filled, with +/// a comment naming this exact hazard. A string is different: only +/// `capacity` bytes of text are ever written by `init_string_header` and its +/// callers' `copy_nonoverlapping`s, so the alignment pad beyond `capacity` +/// -- unlike the array case, invisible to any `StringHeader` field -- is +/// genuinely never initialized. +/// +/// That is harmless to every *string* API: `.length`, indexing, iteration, +/// and every consumer in this crate are bounded by `byte_len`/`capacity`, +/// never `GcHeader.size`. It is NOT harmless to `PERRY_GC_FROMSPACE_SCAN` +/// (`gc/fromspace_scan.rs`), which -- by design, and deliberately consulting +/// no layout state -- trusts `GcHeader.size` as the payload's true extent +/// and scans every word up to it looking for stale from-space references. +/// Leftover bytes from whatever the arena block last held there can, and +/// measurably do, occasionally decode as a plausible NaN-boxed or bare +/// pointer: the #7647 parse-then-churn gate fixture hit this on roughly +/// 1 in 40 parsed-record strings on a clean, correct build, reported as a +/// false "dangling reference" though nothing ever reads that byte range +/// through a real string operation. +/// +/// Zeroing the pad is O(<=7 bytes) per allocation, negligible next to the +/// content copy it sits beside, and makes a string's declared size fully +/// reflect written bytes -- closing the blind spot at this crate's one +/// normal string-storage choke point rather than asking every probe that +/// reaches for the scan to design around it. +#[inline] +fn zero_alignment_padding_tail(raw: *mut u8, requested_payload_size: usize) { + unsafe { + let header = raw.sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; + let allocated_payload = ((*header).size as usize).saturating_sub(crate::gc::GC_HEADER_SIZE); + let padding = allocated_payload.saturating_sub(requested_payload_size); + if padding > 0 { + std::ptr::write_bytes(raw.add(requested_payload_size), 0, padding); + } + } +} + #[inline] pub(crate) unsafe fn init_string_header( ptr: *mut StringHeader, diff --git a/scripts/addr_class_allowlist.txt b/scripts/addr_class_allowlist.txt index 7c0bb1a06f..4365a30627 100644 --- a/scripts/addr_class_allowlist.txt +++ b/scripts/addr_class_allowlist.txt @@ -22,6 +22,7 @@ crates/perry-runtime/src/closure/dynamic_props.rs | for handle in [0x10000usize, # Grandfathered GcHeader-cast files: crates/perry-runtime/src/arena/allocators.rs | * | arena allocator/walker internals: header addresses come from block iteration or fresh allocation, never from NaN-box payloads crates/perry-runtime/src/arena/quarantine.rs | let header = data.add(pos) as *const crate::gc::GcHeader; | #7154 from-space quarantine census: `data + pos` comes from linear block iteration over a detached arena block (the same discipline as arena/walk.rs), never from a NaN-box payload, so no handle band can reach it; the walk stops at the first header whose size does not cover the remaining bytes +crates/perry-runtime/src/string/mod.rs | let header = raw.sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; | #7647 zero_alignment_padding_tail: `raw` is the pointer arena_alloc_gc just returned to string_storage_alloc/_longlived a few lines above, never a NaN-box payload -- same discipline as arena/allocators.rs's own grandfathered entry, reading `.size` back to zero the alignment pad the allocator introduced crates/perry-runtime/src/arena/tests.rs | * | arena allocator/walker internals: header addresses come from block iteration or fresh allocation, never from NaN-box payloads crates/perry-runtime/src/arena/walk.rs | * | arena allocator/walker internals: header addresses come from block iteration or fresh allocation, never from NaN-box payloads crates/perry-runtime/src/array/alloc.rs | * | pre-existing GcHeader probe predating addr_class; address validated by call-site guards (magnitude/registry/is_valid_obj_ptr) -- migrate to addr_class::try_read_gc_header in a follow-up diff --git a/scripts/fixtures/gc_parse_churn_layout_state.ts b/scripts/fixtures/gc_parse_churn_layout_state.ts new file mode 100644 index 0000000000..358d144769 --- /dev/null +++ b/scripts/fixtures/gc_parse_churn_layout_state.ts @@ -0,0 +1,118 @@ +// #7647 end-to-end gate fixture: the "tape=0 + from-space-scan +// parse-then-churn" check for the whole layout-state family +// (#7630 / #7633 / #7635 / #7643 / #7644). +// +// Driven by scripts/gc_parse_churn_layout_gate.sh, never run bare in CI. +// +// SHAPE. `RECORDS` JSON records, each with two pointer-bearing string +// fields. Matches #7643's own reproduction numbers (4,000 records, 2 fields +// -> "dangling=8000 owners=4000" on the sabotaged runtime). Field values are +// unique per record and per field (so any cross-record or cross-field +// corruption is individually detectable) and long enough (>5 chars) to be +// real heap `StringHeader`s rather than inline SSO values. The JSON text is +// a top-level array sized inside the [1 KB, 16 MB) window `js_json_parse`'s +// Auto mode routes through the LAZY TAPE (`json_tape`, default) -- the exact +// shape #7635 found makes a naive parse-then-churn-then-read probe vacuous, +// because the tape defers materializing each record until its first read, +// and a probe that reads only after the churn churns an empty heap. +// +// Each record's two strings are reachable ONLY through that record's own +// field slots -- nothing here keeps a second, parallel reference to them. +// That is #7635's second (discriminating) probe shape: a child reachable +// solely via the slot the collector's layout state is supposed to trace +// strands with nowhere else to be found alive from. +// +// WHAT DEFEATS THE LAZINESS. The gate script runs this fixture with +// `PERRY_JSON_TAPE=0`, which forces every `JSON.parse` call through the +// direct parser regardless of blob size or shape -- the whole record cohort +// is a real materialized tree the moment `JSON.parse` returns, before the +// churn loop below allocates a single byte. This file does not itself prove +// that happened; the gate script does, by cross-checking the from-space +// scan's own `objects=` census against `RECORDS` (a lazily-parsed cohort +// would leave only a handful of tape/lazy-array objects live, not +// thousands of real record objects) -- see that script's header for why +// that is a genuine "was this eager" signal and not merely trust in the +// env var. +// +// WHAT THIS FILE ASSERTS ON ITS OWN. Byte-exact field values after the +// churn: `MISMATCHES` must be `0`. A build that strands a child does not +// necessarily crash (evacuation COPIES rather than zeroes, so a stale +// pointer can still read old, now-reclaimed-and-possibly-reused bytes) -- +// this file's own comparison is the belt to the from-space scan's +// suspenders, and either one failing is real. + +const RECORDS = 4000; + +function fieldA(i: number): string { + return "layout-state-alpha-" + String(i); +} +function fieldB(i: number): string { + return "layout-state-bravo-" + String(i); +} + +function buildBlob(n: number): string { + const parts: string[] = ["["]; + for (let i = 0; i < n; i++) { + if (i > 0) parts.push(","); + parts.push('{"a":"' + fieldA(i) + '","b":"' + fieldB(i) + '"}'); + } + parts.push("]"); + return parts.join(""); +} + +// Parse behind a function boundary so the (potentially large) JSON source +// text is a function-local, not a module-level global. Module-level +// `const`s are registered GC roots for the whole process lifetime (see +// CLAUDE.md's NaN-Boxing section); a function-local's shadow-stack root is +// popped when the function returns, so the source text is ordinary garbage +// by the time the churn loop below runs -- matching how a real "parse once, +// work with the tree" caller is shaped, and keeping the subject of this +// probe to exactly the record cohort the churn should threaten. +function parseRecords(n: number): { a: string; b: string }[] { + const blob = buildBlob(n); + console.log("BLOB_BYTES", blob.length); + return JSON.parse(blob) as { a: string; b: string }[]; +} + +const records = parseRecords(RECORDS); +console.log("PARSED_LENGTH", records.length); + +// Heap churn: enough discarded allocation, across enough separate rounds, to +// force several nursery collections (each round's `garbage` array goes dead +// as soon as the next round starts). Nothing here references `records` -- +// see the shape note above. +const CHURN_ROUNDS = 600; +const CHURN_PER_ROUND = 400; +let churnTouch = 0; +for (let r = 0; r < CHURN_ROUNDS; r++) { + const garbage: { i: number; s: string }[] = []; + for (let i = 0; i < CHURN_PER_ROUND; i++) { + garbage.push({ i: i, s: "churn-filler-" + r + "-" + i }); + } + churnTouch += garbage.length; +} +console.log("CHURN_TOUCH", churnTouch); + +let mismatches = 0; +for (let i = 0; i < RECORDS; i++) { + const rec = records[i]; + if (rec.a !== fieldA(i) || rec.b !== fieldB(i)) { + mismatches++; + if (mismatches <= 5) { + console.log("MISMATCH", i, rec.a, rec.b); + } + } +} +console.log("MISMATCHES", mismatches); + +if (mismatches > 0) { + throw new Error( + "gc_parse_churn_layout_state: " + + mismatches + + " of " + + RECORDS + + " records corrupted after churn" + ); +} + +console.log("PARSE_CHURN_LAYOUT_GATE_OK"); diff --git a/scripts/gc_gate_wiring_check.py b/scripts/gc_gate_wiring_check.py index 01ed14d821..6bb291867c 100644 --- a/scripts/gc_gate_wiring_check.py +++ b/scripts/gc_gate_wiring_check.py @@ -94,6 +94,15 @@ "makes one context speak for all three, so adding an arm later never " "needs a branch-protection edit", ), + ( + ".github/workflows/gc-parse-churn-gate.yml", + "gc-parse-churn-gate", + "scripts/gc_parse_churn_layout_gate.sh — the tape=0 + from-space-scan " + "parse-then-churn end-to-end check for the layout-state family " + "(#7630/#7633/#7635/#7643/#7644), the one shape #7643's workload-free " + "unit tests structurally cannot cover: a NEW materialiser path that " + "forgets to finalize at all", + ), ] MAIN_LINE_EVENTS = ("push", "schedule") diff --git a/scripts/gc_parse_churn_layout_check.py b/scripts/gc_parse_churn_layout_check.py new file mode 100755 index 0000000000..003c49eb86 --- /dev/null +++ b/scripts/gc_parse_churn_layout_check.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +"""Verdict logic for the #7647 parse-then-churn layout-state gate. + +WHY THIS EXISTS +--------------- +#7643 measured that `PERRY_JSON_TAPE=0` + `PERRY_GC_FROMSPACE_SCAN=1` over a +parse-then-churn workload is a known-good end-to-end detector for the whole +layout-state family (#7630 / #7633 / #7635 / #7644): with the JSON +materialiser's finalize sabotaged to always claim `POINTER_FREE`, it reports +`dangling=8000 owners=4000` and the binary SIGBUSes; clean, it reports +`dangling=0` and exits 0. Nothing ran it in CI (#7647). + +Promoting a check to a gate is not just wiring it into a workflow. Per +CLAUDE.md's "four ways a gate can be unable to fail", #4 is the dangerous +one: the gate runs but its subject never does. A parse-then-churn workload +that happens not to trigger a copying minor, or whose JSON stays on the lazy +tape despite `PERRY_JSON_TAPE=0`, would report a clean scan and mean nothing. +So this checker asserts THREE things, not one: + + 1. CORRECTNESS -- the fixture's own byte-exact comparison after the churn + (`MISMATCHES 0`) AND the from-space scan found nothing (no offender + line, which on the gate's own invocation manifests as an aborting + nonzero exit -- see `scripts/gc_parse_churn_layout_gate.sh`). + 2. LIVENESS -- at least one copying minor actually relocated objects + (`copied_objects` summed across every `[gc-copy-minor] ran ...` line is + nonzero). A run that never triggers the moving collector cannot fail + however broken the layout state is. + 3. EAGERNESS -- the from-space scan's own `objects=` census reached at + least `--records` objects. `js_json_parse`'s Auto mode routes a + top-level array in [1 KB, 16 MB) through the LAZY TAPE by default + (json_tape, #7635's whole finding), which defers materializing each + record until first read. `PERRY_JSON_TAPE=0` is supposed to force the + direct (eager) parser for every call regardless of size or shape, but + trusting the env var alone is exactly the kind of assumption #7635 + showed to be worth checking rather than asserting: a lazily-parsed + cohort leaves only a handful of tape/lazy-array objects live at scan + time, nowhere near `--records`. This is the "record count materialised + before the churn, or an equivalent observable" #7647 asks for. + +Usage +----- + python3 scripts/gc_parse_churn_layout_check.py \\ + --exit-code N --stdout stdout.txt --stderr stderr.txt --records 4000 + + python3 scripts/gc_parse_churn_layout_check.py --self-test + +`--self-test` runs the verdict function against synthetic captures covering +every failure mode below, including the vacuous/lazy-tape shape -- proof this +checker can say no, not merely that it has not yet said no. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from dataclasses import dataclass, field + +SUCCESS_SENTINEL = "PARSE_CHURN_LAYOUT_GATE_OK" +MISMATCH_RE = re.compile(r"^MISMATCHES (\d+)$", re.MULTILINE) +COPIED_OBJECTS_RE = re.compile(r"\[gc-copy-minor\] ran copied_objects=(\d+)") +SCAN_LINE_RE = re.compile(r"^\[gc-fromspace-scan (\S+)\] objects=(\d+)", re.MULTILINE) +OFFENDER_PHASES = {"OFFENDERS", "abort"} + + +@dataclass +class Verdict: + ok: bool + problems: list[str] = field(default_factory=list) + + def fail(self, msg: str) -> None: + self.ok = False + self.problems.append(msg) + + +def evaluate(exit_code: int, stdout: str, stderr: str, records: int) -> Verdict: + v = Verdict(ok=True) + + if exit_code != 0: + panic_line = "" + for line in stderr.splitlines(): + if "gc from-space scan" in line or "panicked at" in line: + panic_line = f" ({line.strip()})" + break + v.fail( + f"process exited {exit_code}, expected 0{panic_line}. Under " + f"PERRY_GC_FROMSPACE_SCAN_ABORT=1 a nonzero/signalled exit means " + f"the scan found a surviving from-space reference -- a real " + f"layout-state defect, not a harness problem." + ) + + if SUCCESS_SENTINEL not in stdout: + v.fail( + f"stdout never printed {SUCCESS_SENTINEL!r} -- the fixture did " + f"not run to completion (crashed, threw, or was killed before " + f"its own final assertion)." + ) + + m = MISMATCH_RE.search(stdout) + if m is None: + v.fail("stdout has no 'MISMATCHES ' line -- the fixture did not reach its own read-back check.") + elif int(m.group(1)) != 0: + v.fail( + f"the fixture's own post-churn read-back found {m.group(1)} " + f"corrupted record(s): a record's field value differs from what " + f"was parsed, even though the process did not crash. Evacuation " + f"copies rather than zeroes, so a stranded child can read back " + f"stale-but-plausible bytes without ever faulting -- this is a " + f"real defect the from-space scan can miss." + ) + + copied = [int(n) for n in COPIED_OBJECTS_RE.findall(stderr)] + total_copied = sum(copied) + if total_copied == 0: + v.fail( + "no copying minor relocated anything (sum of every " + "'[gc-copy-minor] ran copied_objects=' line in stderr is 0). " + "The subject of this gate -- the moving collector -- never ran, " + "so a clean scan proves nothing (CLAUDE.md's 'four ways a gate " + "can be unable to fail', #4). Check PERRY_GC_MOVING_LOOP_POLLS=1 " + "was set at BOTH compile time and run time, and PERRY_GC_DIAG=1 " + "at run time so the evidence line is even printed." + ) + + scan_lines = SCAN_LINE_RE.findall(stderr) + if not scan_lines: + v.fail( + "stderr has no '[gc-fromspace-scan ...]' line at all -- the scan " + "itself never ran. PERRY_GC_FROMSPACE_SCAN_ABORT=1 is supposed " + "to imply PERRY_GC_FROMSPACE_SCAN=1 (that used to be false and " + "was fixed; see fromspace_scan.rs's resolve_scan_knobs), so this " + "means either that regressed, or no collection happened at all " + "(see the liveness problem above, if also present)." + ) + else: + max_objects = max(int(n) for _phase, n in scan_lines) + if max_objects < records: + v.fail( + f"the from-space scan's own census topped out at " + f"{max_objects} live objects outside from-space, which is " + f"fewer than the {records} records this fixture parses. " + f"That means the record cohort was not eagerly materialised " + f"before the churn -- PERRY_JSON_TAPE=0 did not force the " + f"direct parser (or something else routed this workload " + f"through the lazy tape), so the from-space scan measured an " + f"empty-ish heap and a clean result would have meant " + f"nothing. This is #7635's original vacuity, recurring." + ) + offender_lines = [ + (phase, n) for phase, n in scan_lines if phase in OFFENDER_PHASES + ] + if offender_lines: + phase, n = offender_lines[0] + v.fail( + f"stderr contains a '[gc-fromspace-scan {phase}]' line " + f"reporting offenders (objects={n}) even though the process " + f"exit code did not reflect it -- inspect the captured " + f"stderr directly." + ) + + return v + + +def format_report(v: Verdict) -> str: + if v.ok: + return "PASS: parse-then-churn layout-state gate is clean and its subject was live." + lines = ["FAIL: parse-then-churn layout-state gate"] + for p in v.problems: + lines.append(f" - {p}") + return "\n".join(lines) + + +# --------------------------------------------------------------------------- self-test + + +def _self_test() -> int: + cases: list[tuple[str, int, str, str, int, bool]] = [] + + ok_stdout = "BLOB_BYTES 245781\nPARSED_LENGTH 4000\nCHURN_TOUCH 240000\nMISMATCHES 0\n" + SUCCESS_SENTINEL + "\n" + ok_stderr = ( + "[gc-copy-minor] ran copied_objects=537 copied_bytes=46112\n" + "[gc-fromspace-scan clean] objects=6041 words=86354 fwd_owners_skipped=0 missing_rewrites=0 dangling=0 owners=0\n" + "[gc-copy-minor] ran copied_objects=612 copied_bytes=51200\n" + "[gc-fromspace-scan clean] objects=9210 words=120000 fwd_owners_skipped=0 missing_rewrites=0 dangling=0 owners=0\n" + ) + cases.append(("clean run passes", 0, ok_stdout, ok_stderr, 4000, True)) + + aborted_stderr = ok_stderr + ( + "[gc-fromspace-scan abort] objects=5052 words=86354 fwd_owners_skipped=3 " + "missing_rewrites=0 dangling=1 owners=1\n" + "thread '' panicked at crates/perry-runtime/src/gc/fromspace_scan.rs:351:5:\n" + "gc from-space scan: 0 missing rewrite(s), 1 dangling reference(s) survived the rewrite pass\n" + ) + truncated_stdout = "BLOB_BYTES 245781\nPARSED_LENGTH 4000\n" + cases.append(("real dangling reference aborts the process -> FAIL", 134, truncated_stdout, aborted_stderr, 4000, False)) + + corrupted_stdout = ok_stdout.replace("MISMATCHES 0", "MISMATCHES 3").replace( + SUCCESS_SENTINEL + "\n", "" + ) + cases.append(("silent corruption without a crash -> FAIL", 1, corrupted_stdout, ok_stderr, 4000, False)) + + no_copy_stderr = "\n".join( + line for line in ok_stderr.splitlines() if "gc-copy-minor" not in line + ) + cases.append(("no copying minor ever ran -> FAIL (liveness)", 0, ok_stdout, no_copy_stderr, 4000, False)) + + no_scan_stderr = "\n".join( + line for line in ok_stderr.splitlines() if "gc-fromspace-scan" not in line + ) + cases.append(("scan never ran at all -> FAIL (ABORT-alone-inert class)", 0, ok_stdout, no_scan_stderr, 4000, False)) + + lazy_stderr = ( + "[gc-copy-minor] ran copied_objects=4 copied_bytes=512\n" + "[gc-fromspace-scan clean] objects=9 words=88 fwd_owners_skipped=0 missing_rewrites=0 dangling=0 owners=0\n" + ) + cases.append(("tape stayed lazy: scan sees ~9 objects not 4000 -> FAIL (eagerness/vacuity)", 0, ok_stdout, lazy_stderr, 4000, False)) + + missing_sentinel_stdout = "BLOB_BYTES 245781\nPARSED_LENGTH 4000\nMISMATCHES 0\n" + cases.append(("crashed after the mismatch check, before the sentinel -> FAIL", 1, missing_sentinel_stdout, ok_stderr, 4000, False)) + + failures = [] + for name, exit_code, stdout, stderr, records, expect_ok in cases: + v = evaluate(exit_code, stdout, stderr, records) + if v.ok != expect_ok: + failures.append( + f" - {name}: expected ok={expect_ok}, got ok={v.ok} ({v.problems})" + ) + else: + print(f"ok {name}") + + if failures: + print("SELF-TEST FAILED:") + for f in failures: + print(f) + return 1 + print(f"\nSELF-TEST OK: {len(cases)} cases, including both directions " + "(a real defect fails it; a clean+live run passes it).") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--self-test", action="store_true") + parser.add_argument("--exit-code", type=int) + parser.add_argument("--stdout", help="path to captured stdout") + parser.add_argument("--stderr", help="path to captured stderr") + parser.add_argument("--records", type=int, help="expected record-cohort size") + args = parser.parse_args() + + if args.self_test: + return _self_test() + + missing = [ + name + for name, val in ( + ("--exit-code", args.exit_code), + ("--stdout", args.stdout), + ("--stderr", args.stderr), + ("--records", args.records), + ) + if val is None + ] + if missing: + parser.error(f"missing required argument(s): {', '.join(missing)} (or pass --self-test)") + + with open(args.stdout, encoding="utf-8", errors="replace") as f: + stdout_text = f.read() + with open(args.stderr, encoding="utf-8", errors="replace") as f: + stderr_text = f.read() + + v = evaluate(args.exit_code, stdout_text, stderr_text, args.records) + print(format_report(v)) + return 0 if v.ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/gc_parse_churn_layout_gate.sh b/scripts/gc_parse_churn_layout_gate.sh new file mode 100755 index 0000000000..e6a7cc126b --- /dev/null +++ b/scripts/gc_parse_churn_layout_gate.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# The #7647 CI gate: promotes the "tape=0 + from-space-scan parse-then-churn" +# check (#7643's own follow-up note, descending from #7635) from a hand-run +# investigation into something CI runs on every relevant change. +# +# WHY THIS EXISTS +# +# `PERRY_JSON_TAPE=0` + `PERRY_GC_FROMSPACE_SCAN=1` over a parse-then-churn +# workload is a known-good end-to-end detector for the whole layout-state +# family: with the JSON materialiser's finalize sabotaged to always claim +# `POINTER_FREE` (#7635's exact mutation), it reports `dangling=8000 +# owners=4000` and the binary SIGBUSes; clean, `dangling=0` and exit 0. +# #7643/#7644 shipped unit tests for two invariants that need no workload at +# all (the child-slot enumerator, and relocation of a hand-built sabotaged +# record) -- the right primary guard, because they cannot be defeated by a +# lazy path or a GC that did not happen to run. But neither can catch a NEW +# materialiser path that forgets to finalize at all, since such a path would +# simply not be exercised by a hand-built object. That is what this +# end-to-end gate is for, and until now nothing ran it in CI. +# +# THIS SCRIPT IS DESIGNED TO BE ABLE TO FAIL, checked against CLAUDE.md's +# "four ways a gate can be unable to fail": +# +# 1. no `continue-on-error`, no `|| true`, no pipe between a checker and +# the shell's exit status -- see the final `exit` below. +# 2. NOT wired into branch protection's required contexts by this change. +# A new gate has never been green, so promoting it immediately would +# block every open PR -- that is a maintainer action for after one +# observed green run on `main` (CLAUDE.md's corollary: the promotion +# step must actually be taken, not left undone). +# 3. the workflow's `concurrency` block cancels pull_request runs only; +# push (main) runs are keyed by commit SHA so they queue instead of +# cancelling each other (see .github/workflows/gc-parse-churn-gate.yml). +# 4. the subject must be LIVE, not merely quiet. `PERRY_GC_FROMSPACE_SCAN` +# only ever runs during a COPYING minor, and the moving collector is +# opt-in (`PERRY_GC_MOVING_LOOP_POLLS=1`, both at compile time and run +# time -- #7161's stopgap made this the ONLY configuration that +# exercises it end to end). And `PERRY_JSON_TAPE=0` alone is a knob +# setting, not proof it was honoured. So `scripts/gc_parse_churn_layout_check.py` +# -- the actual pass/fail logic, invoked below -- rejects a run with +# zero copying minors AND a run whose from-space census stayed small +# enough that the record cohort was plainly still on the lazy tape. +# Both are read from the SAME captured output the correctness check +# uses, so there is no separate "did it run" side-channel to drift out +# of sync. +# +# Usage: scripts/gc_parse_churn_layout_gate.sh [path-to-perry] +# Expects a `perry` binary whose PERRY_RUNTIME_DIR-resolvable staticlibs +# are current (see CLAUDE.md's "Verifying a runtime change" pitfall -- +# perry-runtime/perry-stdlib are rlib-only, the .a comes from the +# *-static wrapper crates, and a stale archive makes this whole gate +# vacuous in a way nothing here can detect). + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PERRY_BIN="${1:-$REPO_ROOT/target/release/perry}" +if [[ ! -x "$PERRY_BIN" ]]; then + echo "FAIL: no perry binary at $PERRY_BIN" >&2 + exit 1 +fi +PERRY_BIN="$(cd "$(dirname "$PERRY_BIN")" && pwd)/$(basename "$PERRY_BIN")" +export PERRY_RUNTIME_DIR="${PERRY_RUNTIME_DIR:-$(dirname "$PERRY_BIN")}" +# Ad-hoc compiles must not link a per-app auto-optimized (feature-stripped) +# runtime: the diagnostics this gate reads (`[gc-copy-minor]`, +# `[gc-fromspace-scan ...]`) are ordinary env-gated eprintln!s (not behind +# the `diagnostics` cargo feature -- that name collision is with Node's +# `diagnostics_channel` support, a different thing), but auto-optimize can +# still relink against a differently-built stdlib/runtime pair than the one +# just built. Pin it explicitly rather than rely on that being harmless. +export PERRY_NO_AUTO_OPTIMIZE=1 + +FIXTURE="$REPO_ROOT/scripts/fixtures/gc_parse_churn_layout_state.ts" +if [[ ! -f "$FIXTURE" ]]; then + echo "FAIL: fixture not found at $FIXTURE" >&2 + exit 1 +fi + +# Keep the gate's expected record count in sync with the fixture by reading +# it out, rather than hand-copying the number into this script (and letting +# it silently drift the next time someone retunes the fixture). +RECORDS="$(grep -oE '^const RECORDS = [0-9]+;' "$FIXTURE" | grep -oE '[0-9]+')" +if [[ -z "$RECORDS" ]]; then + echo "FAIL: could not read 'const RECORDS = N;' out of $FIXTURE" >&2 + exit 1 +fi + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +echo "== compiling $FIXTURE (PERRY_GC_MOVING_LOOP_POLLS=1, records=$RECORDS) ==" +# PERRY_GC_MOVING_LOOP_POLLS is a COMPILE-TIME gate as well as a runtime one +# (perry-codegen's moving_safepoint_polls_enabled() decides whether loop +# back-edge polls are even emitted) -- omitting it here would run the fixture +# below at safepoints that were never emitted, which the moving collector +# would then never reach without falling back to a conservative-stack-scan +# collection. A conservative-scan cycle is ineligible for the copying path +# (`CopiedMinorFallbackReason::ConservativeStack`), so it would run non-moving +# full mark-sweeps for the whole churn and never trigger the from-space scan +# at all -- the liveness check below exists exactly to catch that shape. +PERRY_GC_MOVING_LOOP_POLLS=1 "$PERRY_BIN" compile "$FIXTURE" -o "$WORK/fixture" >/dev/null + +echo "== running under PERRY_JSON_TAPE=0 + PERRY_GC_FROMSPACE_SCAN_ABORT=1 ==" +set +e +PERRY_GC_MOVING_LOOP_POLLS=1 \ +PERRY_JSON_TAPE=0 \ +PERRY_GC_FROMSPACE_SCAN_ABORT=1 \ +PERRY_GC_DIAG=1 \ +PERRY_GC_HEAP_LIMIT="${PERRY_GC_HEAP_LIMIT:-8}" \ + "$WORK/fixture" >"$WORK/stdout.txt" 2>"$WORK/stderr.txt" +RC=$? +set -e + +echo "-- stdout --" +cat "$WORK/stdout.txt" +echo "-- stderr (tail) --" +tail -20 "$WORK/stderr.txt" +echo "-- exit code: $RC --" + +python3 "$REPO_ROOT/scripts/gc_parse_churn_layout_check.py" \ + --exit-code "$RC" \ + --stdout "$WORK/stdout.txt" \ + --stderr "$WORK/stderr.txt" \ + --records "$RECORDS" +CHECK_RC=$? + +exit "$CHECK_RC"