diff --git a/.github/workflows/tls-budget.yml b/.github/workflows/tls-budget.yml new file mode 100644 index 0000000000..ca79934df2 --- /dev/null +++ b/.github/workflows/tls-budget.yml @@ -0,0 +1,171 @@ +name: TLS Budget + +# Keeps Darwin's `_tlv_get_addr` cost from creeping back into the runtime +# a fourth time (#7469). +# +# WHY THIS EXISTS +# +# On Darwin every `thread_local!` access is an out-of-line call to +# `_tlv_get_addr` in libdyld. `crates/perry-runtime/src/tls_hot.rs` has removed +# that cost three times and it has come back three times: +# +# after #7565 churn_alloc 0% +# later churn_alloc 8-9% +# later interp 11% +# v0.5.1434 asyncpipe 20.5% <- the largest single symbol in the +# worst-performing realistic program +# +# The mechanism never decayed. Nothing measured it. This job is that +# measurement. +# +# THIS JOB IS DESIGNED TO BE ABLE TO FAIL, checked against CLAUDE.md's "four +# ways a gate can be unable to fail": +# +# 1. no `continue-on-error`, no `|| true` between the gate script and the +# shell's exit status. `scripts/tls_budget_gate.sh` runs under +# `set -euo pipefail` and ends in a bare `exit "$rc"`. +# 2. NOT wired into branch protection's required contexts by the change that +# adds it -- a new gate has never been green, so promoting it immediately +# would block every open PR (CLAUDE.md's corollary). That is a maintainer +# action after the first observed green run on `main`, and per the +# corollary it is not optional follow-through: `gc-root-dominance` sat red +# on `main` for weeks because the second step was never taken. +# 3. `concurrency` below cancels `pull_request` runs only; `push` runs are +# keyed on the commit SHA so they queue instead of cancelling each other +# (#7205). +# 4. THE SUBJECT MUST BE THE UNCOVERED ONE, and that is the whole design. +# Profiling `churn_alloc` -- the benchmark every previous fix was tuned +# against -- would pass forever while the real cost grew, because churn's +# thread-locals are exactly the sixteen the named-field cache covers by +# construction. So the subjects are `benchmarks/tls-budget/asyncpipe.ts` +# (Map/Set registries, buffer brands, descriptor state, async, template +# literals) and `interp.ts` (inline-cache misses, field lookup, arguments +# objects), and `scripts/tls_budget_check.py` refuses a pass unless the +# run proves it was live: `PERRY_TLS_HOT_STATS=1` reporting +# `direct_tsd=1` (else `hot()` is itself calling `_tlv_get_addr` and the +# mechanism is inert) and `claimed` above a floor no allocation +# microbenchmark clears. Its `--self-test` drives all seven rejections +# and runs on every PR, compiler-free, in the first job below. +# +# The one-time sabotage proof -- one hot declaration reverted to a raw +# `thread_local!`, both budgets going red, restoring the macro restoring the +# pass -- is recorded in the PR that introduced this file, matching how +# gc-root-dominance and gc-parse-churn-gate treat theirs. + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: tls-budget-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CARGO_TERM_COLOR: always + MACOSX_DEPLOYMENT_TARGET: "13.0" + +jobs: + self-test-checkers: + # Compiler-free and unconditional: the verdict logic must always be able to + # say no, and the thread-local policy ratchet is cheap enough to run on + # every PR regardless of what it touched. + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Self-test the budget verdict logic + run: python3 scripts/tls_budget_check.py --self-test + - name: Self-test the thread-local policy checker + run: python3 scripts/check_thread_locals.py --self-test + - name: Enforce the thread-local policy ratchet + run: python3 scripts/check_thread_locals.py + + tls-budget: + # macos-14 is arm64. `_tlv_get_addr` is a Mach-O TLS artefact and the + # direct thread-specific-data path in tls_hot.rs is Apple-aarch64 only, so + # this measurement does not exist on any other platform -- the gate script + # says so and exits 0 rather than pretending to measure. + runs-on: macos-14 + timeout-minutes: 90 + permissions: + contents: read + pull-requests: read + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Decide whether this change can affect thread-local access cost + id: relevance + env: + GH_TOKEN: ${{ github.token }} + EVENT_NAME: ${{ github.event_name }} + REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + set -euo pipefail + if [[ "$EVENT_NAME" != "pull_request" ]]; then + echo "run=true" >> "$GITHUB_OUTPUT" + echo "Not a pull request; running the gate." + exit 0 + fi + gh api "repos/$REPOSITORY/pulls/$PR_NUMBER/files" --paginate --jq '.[].filename' > changed.txt + # Deliberately broad: any runtime change can add a thread-local to a + # hot path, and the whole point of this gate is that such a change + # does not announce itself. The filter exists only to spare docs-only + # PRs a compiler build. + if grep -qE '^(crates/|benchmarks/tls-budget/|scripts/tls_budget_(gate\.sh|check\.py)$|scripts/check_thread_locals\.py$|Cargo\.(toml|lock)$|\.github/workflows/tls-budget\.yml$)' changed.txt; then + echo "run=true" >> "$GITHUB_OUTPUT" + echo "Change touches a path that can move thread-local access cost." + else + echo "run=false" >> "$GITHUB_OUTPUT" + echo "No relevant paths changed." + fi + + - name: Install Rust toolchain + if: steps.relevance.outputs.run == 'true' + uses: dtolnay/rust-toolchain@stable + - uses: ./.github/actions/setup-llvm22 + if: steps.relevance.outputs.run == 'true' + + - uses: Swatinem/rust-cache@v2 + if: steps.relevance.outputs.run == 'true' + with: + shared-key: "${{ runner.os }}-perry" + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Build perry and the runtime archives + if: steps.relevance.outputs.run == 'true' + run: | + set -euo pipefail + # perry-runtime and perry-stdlib are rlib-only; libperry_runtime.a + # and libperry_stdlib.a come from the -static wrapper crates. Building + # without them links a stale archive and makes the measurement + # vacuous (CLAUDE.md, "Verifying a runtime change"). + cargo build --release -p perry -p perry-runtime-static -p perry-stdlib-static + for artifact in perry libperry_runtime.a libperry_stdlib.a; do + test -s "target/release/$artifact" \ + || { echo "::error::target/release/$artifact was not produced"; exit 1; } + done + + - name: Measure the `_tlv_get_addr` budget + if: steps.relevance.outputs.run == 'true' + env: + PERRY_RUNTIME_DIR: ${{ github.workspace }}/target/release + PERRY_NO_AUTO_OPTIMIZE: "1" + run: scripts/tls_budget_gate.sh target/release/perry "${{ runner.temp }}/tls-budget" + + - name: Attach the profiles + if: always() && steps.relevance.outputs.run == 'true' + uses: actions/upload-artifact@v4 + with: + name: tls-budget-profiles + path: ${{ runner.temp }}/tls-budget/*.sample + if-no-files-found: warn diff --git a/CLAUDE.md b/CLAUDE.md index 4675051f29..69d27ca3e3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Perry is a native TypeScript compiler written in Rust that compiles TypeScript source code directly to native executables. It uses SWC for TypeScript parsing and LLVM for code generation. -**Current Version:** 0.5.1443 +**Current Version:** 0.5.1444 ## TypeScript Parity Status diff --git a/Cargo.lock b/Cargo.lock index 8721032f14..b9ce96cf0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5547,7 +5547,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "perry" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "anyhow", "base64", @@ -5607,14 +5607,14 @@ dependencies = [ [[package]] name = "perry-api-manifest" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "serde", ] [[package]] name = "perry-audio-miniaudio" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "cc", "libc", @@ -5622,7 +5622,7 @@ dependencies = [ [[package]] name = "perry-codegen" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "anyhow", "inkwell", @@ -5639,7 +5639,7 @@ dependencies = [ [[package]] name = "perry-codegen-arkts" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "anyhow", "perry-hir", @@ -5647,7 +5647,7 @@ dependencies = [ [[package]] name = "perry-codegen-glance" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "anyhow", "perry-hir", @@ -5655,7 +5655,7 @@ dependencies = [ [[package]] name = "perry-codegen-js" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "anyhow", "perry-dispatch", @@ -5664,7 +5664,7 @@ dependencies = [ [[package]] name = "perry-codegen-swiftui" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "anyhow", "perry-hir", @@ -5672,7 +5672,7 @@ dependencies = [ [[package]] name = "perry-codegen-wasm" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "anyhow", "base64", @@ -5684,7 +5684,7 @@ dependencies = [ [[package]] name = "perry-codegen-wear-tiles" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "anyhow", "perry-hir", @@ -5692,7 +5692,7 @@ dependencies = [ [[package]] name = "perry-container-compose" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "anyhow", "async-trait", @@ -5721,14 +5721,14 @@ dependencies = [ [[package]] name = "perry-container-e2e" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "anyhow", ] [[package]] name = "perry-diagnostics" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "serde", "serde_json", @@ -5736,7 +5736,7 @@ dependencies = [ [[package]] name = "perry-dispatch" -version = "0.5.1443" +version = "0.5.1444" [[package]] name = "perry-doc-fixture-my-bindings" @@ -5747,7 +5747,7 @@ dependencies = [ [[package]] name = "perry-doc-tests" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "anyhow", "clap", @@ -5762,7 +5762,7 @@ dependencies = [ [[package]] name = "perry-ext-ads" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "block2", "objc2", @@ -5772,7 +5772,7 @@ dependencies = [ [[package]] name = "perry-ext-argon2" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "argon2", "perry-ffi", @@ -5780,7 +5780,7 @@ dependencies = [ [[package]] name = "perry-ext-axios" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "perry-ffi", "reqwest", @@ -5789,7 +5789,7 @@ dependencies = [ [[package]] name = "perry-ext-bcrypt" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "bcrypt", "perry-ffi", @@ -5797,7 +5797,7 @@ dependencies = [ [[package]] name = "perry-ext-better-sqlite3" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "perry-ffi", "rusqlite", @@ -5805,7 +5805,7 @@ dependencies = [ [[package]] name = "perry-ext-cheerio" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "perry-ffi", "scraper", @@ -5813,7 +5813,7 @@ dependencies = [ [[package]] name = "perry-ext-commander" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "perry-ffi", "perry-runtime", @@ -5821,7 +5821,7 @@ dependencies = [ [[package]] name = "perry-ext-cron" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "chrono", "cron", @@ -5831,7 +5831,7 @@ dependencies = [ [[package]] name = "perry-ext-dayjs" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "chrono", "perry-ffi", @@ -5839,7 +5839,7 @@ dependencies = [ [[package]] name = "perry-ext-decimal" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "perry-ffi", "rust_decimal", @@ -5847,7 +5847,7 @@ dependencies = [ [[package]] name = "perry-ext-dotenv" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "perry-ffi", "serde_json", @@ -5855,7 +5855,7 @@ dependencies = [ [[package]] name = "perry-ext-ethers" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "perry-ffi", "rand 0.10.1", @@ -5863,7 +5863,7 @@ dependencies = [ [[package]] name = "perry-ext-events" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "perry-ffi", "perry-runtime", @@ -5871,14 +5871,14 @@ dependencies = [ [[package]] name = "perry-ext-exponential-backoff" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "perry-ffi", ] [[package]] name = "perry-ext-fastify" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "bytes", "http-body-util", @@ -5896,7 +5896,7 @@ dependencies = [ [[package]] name = "perry-ext-fetch" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "bytes", "lazy_static", @@ -5909,7 +5909,7 @@ dependencies = [ [[package]] name = "perry-ext-http" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "bytes", "h2", @@ -5933,7 +5933,7 @@ dependencies = [ [[package]] name = "perry-ext-ioredis" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "lazy_static", "perry-ffi", @@ -5943,7 +5943,7 @@ dependencies = [ [[package]] name = "perry-ext-jsonwebtoken" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "base64", "jsonwebtoken", @@ -5954,7 +5954,7 @@ dependencies = [ [[package]] name = "perry-ext-lru-cache" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "lru", "perry-ffi", @@ -5963,7 +5963,7 @@ dependencies = [ [[package]] name = "perry-ext-moment" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "chrono", "perry-ffi", @@ -5971,7 +5971,7 @@ dependencies = [ [[package]] name = "perry-ext-mongodb" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "bson", "futures-util", @@ -5983,7 +5983,7 @@ dependencies = [ [[package]] name = "perry-ext-mysql2" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "chrono", "perry-ffi", @@ -5993,7 +5993,7 @@ dependencies = [ [[package]] name = "perry-ext-nanoid" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "nanoid", "perry-ffi", @@ -6002,7 +6002,7 @@ dependencies = [ [[package]] name = "perry-ext-net" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "bytes", "perry-ffi", @@ -6015,7 +6015,7 @@ dependencies = [ [[package]] name = "perry-ext-node-forge" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "const-oid 0.9.6", "der 0.7.10", @@ -6034,7 +6034,7 @@ dependencies = [ [[package]] name = "perry-ext-nodemailer" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "lettre", "perry-ffi", @@ -6044,7 +6044,7 @@ dependencies = [ [[package]] name = "perry-ext-pdf" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "perry-ffi", "printpdf", @@ -6052,7 +6052,7 @@ dependencies = [ [[package]] name = "perry-ext-pg" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "perry-ffi", "sqlx", @@ -6061,7 +6061,7 @@ dependencies = [ [[package]] name = "perry-ext-ratelimit" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "governor", "perry-ffi", @@ -6069,7 +6069,7 @@ dependencies = [ [[package]] name = "perry-ext-sharp" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "fast_image_resize", "image", @@ -6079,14 +6079,14 @@ dependencies = [ [[package]] name = "perry-ext-slugify" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "perry-ffi", ] [[package]] name = "perry-ext-streams" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "lazy_static", "perry-ffi", @@ -6095,7 +6095,7 @@ dependencies = [ [[package]] name = "perry-ext-undici" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "perry-ffi", "perry-runtime", @@ -6104,7 +6104,7 @@ dependencies = [ [[package]] name = "perry-ext-uuid" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "perry-ffi", "uuid", @@ -6112,7 +6112,7 @@ dependencies = [ [[package]] name = "perry-ext-validator" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "perry-ffi", "regex", @@ -6122,7 +6122,7 @@ dependencies = [ [[package]] name = "perry-ext-ws" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "futures-util", "lazy_static", @@ -6135,7 +6135,7 @@ dependencies = [ [[package]] name = "perry-ext-zlib" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "brotli", "flate2", @@ -6145,7 +6145,7 @@ dependencies = [ [[package]] name = "perry-ffi" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "dashmap", "once_cell", @@ -6154,7 +6154,7 @@ dependencies = [ [[package]] name = "perry-hir" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "anyhow", "perry-api-manifest", @@ -6172,7 +6172,7 @@ dependencies = [ [[package]] name = "perry-parser" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "anyhow", "perry-diagnostics", @@ -6184,7 +6184,7 @@ dependencies = [ [[package]] name = "perry-runtime" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "anyhow", "base64", @@ -6226,14 +6226,14 @@ dependencies = [ [[package]] name = "perry-runtime-static" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "perry-runtime", ] [[package]] name = "perry-stdlib" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "aes 0.8.4", "aes 0.9.1", @@ -6328,14 +6328,14 @@ dependencies = [ [[package]] name = "perry-stdlib-static" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "perry-stdlib", ] [[package]] name = "perry-transform" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "anyhow", "perry-hir", @@ -6344,14 +6344,14 @@ dependencies = [ [[package]] name = "perry-ui" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "perry-ui-model", ] [[package]] name = "perry-ui-android" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "base64", "itoa", @@ -6368,7 +6368,7 @@ dependencies = [ [[package]] name = "perry-ui-geisterhand" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "rand 0.10.1", "serde", @@ -6378,7 +6378,7 @@ dependencies = [ [[package]] name = "perry-ui-gtk4" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "base64", "cairo-rs 0.22.0", @@ -6401,7 +6401,7 @@ dependencies = [ [[package]] name = "perry-ui-ios" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "base64", "block2", @@ -6417,7 +6417,7 @@ dependencies = [ [[package]] name = "perry-ui-macos" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "base64", "block2", @@ -6432,7 +6432,7 @@ dependencies = [ [[package]] name = "perry-ui-model" -version = "0.5.1443" +version = "0.5.1444" [[package]] name = "perry-ui-test" @@ -6443,11 +6443,11 @@ dependencies = [ [[package]] name = "perry-ui-testkit" -version = "0.5.1443" +version = "0.5.1444" [[package]] name = "perry-ui-tvos" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "base64", "block2", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "perry-ui-visionos" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "base64", "block2", @@ -6479,7 +6479,7 @@ dependencies = [ [[package]] name = "perry-ui-watchos" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "block2", "libc", @@ -6492,7 +6492,7 @@ dependencies = [ [[package]] name = "perry-ui-windows" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "base64", "libc", @@ -6509,14 +6509,14 @@ dependencies = [ [[package]] name = "perry-ui-windows-winui" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "perry-ui-windows", ] [[package]] name = "perry-updater" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "anyhow", "base64", @@ -6532,7 +6532,7 @@ dependencies = [ [[package]] name = "perry-wasm-host" -version = "0.5.1443" +version = "0.5.1444" dependencies = [ "wasmi", ] diff --git a/Cargo.toml b/Cargo.toml index ffcbca7348..d13bf24575 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -315,7 +315,7 @@ codegen-units = 16 codegen-units = 16 [workspace.package] -version = "0.5.1443" +version = "0.5.1444" edition = "2021" license = "MIT" repository = "https://github.com/PerryTS/perry" diff --git a/benchmarks/tls-budget/asyncpipe.ts b/benchmarks/tls-budget/asyncpipe.ts new file mode 100644 index 0000000000..a78ddc1deb --- /dev/null +++ b/benchmarks/tls-budget/asyncpipe.ts @@ -0,0 +1,113 @@ +// An async request-handling pipeline: the shape of a small service. +// Batches of "requests" flow through validate -> enrich -> aggregate, with +// awaits between stages, Promise.all fan-out per batch, a Map-backed index, +// template-literal log lines, and a timer yield at each batch boundary. +// +// Deliberately covers four things the benchmark corpus never touched: +// async/await + promise combinators, Map/Set, template literals, and +// closures created inside loops (a documented weak spot in the +// async-to-generator transform: per-iteration `let` bindings collapsing). +// +// Fully deterministic: no clock reads, no randomness, fixed inputs. + +type Req = { id: number; user: string; kind: string; amount: number }; +type Ok = { id: number; user: string; amount: number; note: string }; + +const KINDS: string[] = ["order", "refund", "adjust"]; +const USERS: string[] = ["ana", "bo", "cyd", "dee", "eli", "fay"]; + +function makeReq(i: number): Req { + return { + id: i, + user: USERS[i % USERS.length], + kind: KINDS[i % KINDS.length], + amount: (i % 97) + 1, + }; +} + +function tick(): Promise { + return new Promise((resolve) => { + setTimeout(() => resolve(0), 0); + }); +} + +async function validate(r: Req): Promise { + if (r.amount <= 0) throw new Error(`bad amount for ${r.id}`); + return r; +} + +async function enrich(r: Req, rates: Map): Promise { + const rate = rates.has(r.kind) ? (rates.get(r.kind) as number) : 1; + const scaled = r.amount * rate; + return { + id: r.id, + user: r.user, + amount: scaled, + note: `${r.kind}:${r.user}#${r.id}=${scaled}`, + }; +} + +async function handle(r: Req, rates: Map): Promise { + const v = await validate(r); + const e = await enrich(v, rates); + return e; +} + +async function runBatch(base: number, size: number, rates: Map): Promise { + // Closures created inside a loop, each capturing a per-iteration binding. + const jobs: Promise[] = []; + for (let k = 0; k < size; k++) { + const req = makeReq(base + k); + jobs.push(handle(req, rates)); + } + const done: Ok[] = await Promise.all(jobs); + return done; +} + +async function main(): Promise { + const rates = new Map(); + rates.set("order", 3); + rates.set("refund", 5); + rates.set("adjust", 2); + + const seenUsers = new Set(); + const perUser = new Map(); + + let total = 0; + let noteLen = 0; + let errors = 0; + + const BATCHES = 1200; + const SIZE = 200; + + for (let b = 0; b < BATCHES; b++) { + let batch: Ok[] = []; + try { + batch = await runBatch(b * SIZE, SIZE, rates); + } catch (e) { + errors = errors + 1; + batch = []; + } + for (let i = 0; i < batch.length; i++) { + const ok = batch[i]; + total = total + ok.amount; + noteLen = noteLen + ok.note.length; + seenUsers.add(ok.user); + const prev = perUser.has(ok.user) ? (perUser.get(ok.user) as number) : 0; + perUser.set(ok.user, prev + ok.amount); + } + if (b % 40 === 0) { + await tick(); + } + } + + let userSum = 0; + for (let u = 0; u < USERS.length; u++) { + const name = USERS[u]; + userSum = userSum + (perUser.has(name) ? (perUser.get(name) as number) : 0); + } + + console.log(`${total} ${noteLen} ${seenUsers.size} ${userSum} ${errors}`); +} + +main(); diff --git a/benchmarks/tls-budget/interp.ts b/benchmarks/tls-budget/interp.ts new file mode 100644 index 0000000000..6b282110e8 --- /dev/null +++ b/benchmarks/tls-budget/interp.ts @@ -0,0 +1,283 @@ +// A tree-walking interpreter for a small functional language. +// Written to stay inside scriptc's STATIC tier (no generics, no `any`, +// no `var`/`==`, dense arrays, string-keyed Maps, discriminated unions). +// Exercises: recursive union allocation, closures capturing environments, +// polymorphic dispatch, string building, deep recursion, Map lookups. + +type Node = + | { kind: "num"; num: number } + | { kind: "str"; str: string } + | { kind: "var"; name: string } + | { kind: "bin"; op: string; left: Node; right: Node } + | { kind: "if"; cond: Node; then: Node; alt: Node } + | { kind: "let"; name: string; value: Node; body: Node } + | { kind: "fun"; param: string; body: Node } + | { kind: "call"; target: Node; arg: Node }; + +type Env = { names: string[]; vals: Value[]; parent: Env | null }; + +type Value = + | { kind: "num"; num: number } + | { kind: "str"; str: string } + | { kind: "clo"; param: string; body: Node; env: Env }; + +// ---------- lexer ---------- + +type Token = { kind: string; text: string }; + +function isDigit(c: string): boolean { + return c >= "0" && c <= "9"; +} + +function isIdentStart(c: string): boolean { + return (c >= "a" && c <= "z") || (c >= "A" && c <= "Z") || c === "_"; +} + +function isIdentPart(c: string): boolean { + return isIdentStart(c) || isDigit(c); +} + +function lex(src: string): Token[] { + const out: Token[] = []; + let i = 0; + while (i < src.length) { + const c = src.charAt(i); + if (c === " " || c === "\n" || c === "\t") { + i = i + 1; + continue; + } + if (isDigit(c)) { + let j = i; + while (j < src.length && isDigit(src.charAt(j))) j = j + 1; + out.push({ kind: "num", text: src.substring(i, j) }); + i = j; + continue; + } + if (isIdentStart(c)) { + let j = i; + while (j < src.length && isIdentPart(src.charAt(j))) j = j + 1; + const word = src.substring(i, j); + if (word === "let" || word === "in" || word === "if" || word === "then" || word === "else" || word === "fun") { + out.push({ kind: word, text: word }); + } else { + out.push({ kind: "ident", text: word }); + } + i = j; + continue; + } + if (c === '"') { + let j = i + 1; + while (j < src.length && src.charAt(j) !== '"') j = j + 1; + out.push({ kind: "str", text: src.substring(i + 1, j) }); + i = j + 1; + continue; + } + if (c === "<" && i + 1 < src.length && src.charAt(i + 1) === "=") { + out.push({ kind: "op", text: "<=" }); + i = i + 2; + continue; + } + out.push({ kind: c === "(" || c === ")" ? c : "op", text: c }); + i = i + 1; + } + out.push({ kind: "eof", text: "" }); + return out; +} + +// ---------- parser (precedence climbing) ---------- + +type Parser = { toks: Token[]; pos: number }; + +function peek(p: Parser): Token { + return p.toks[p.pos]; +} + +function advance(p: Parser): Token { + const t = p.toks[p.pos]; + p.pos = p.pos + 1; + return t; +} + +function expect(p: Parser, kind: string): void { + const t = advance(p); + if (t.kind !== kind) { + console.log("parse error: wanted " + kind + " got " + t.kind); + } +} + +function precOf(op: string): number { + if (op === "<" || op === ">" || op === "<=") return 1; + if (op === "+" || op === "-") return 2; + if (op === "*" || op === "/") return 3; + return 0; +} + +function parseAtom(p: Parser): Node { + const t = peek(p); + if (t.kind === "num") { + advance(p); + return { kind: "num", num: parseInt(t.text, 10) }; + } + if (t.kind === "str") { + advance(p); + return { kind: "str", str: t.text }; + } + if (t.kind === "ident") { + advance(p); + return { kind: "var", name: t.text }; + } + if (t.kind === "(") { + advance(p); + const inner = parseExpr(p, 0); + expect(p, ")"); + return inner; + } + if (t.kind === "fun") { + advance(p); + const name = advance(p); + const body = parseExpr(p, 0); + return { kind: "fun", param: name.text, body: body }; + } + if (t.kind === "let") { + advance(p); + const name = advance(p); + expect(p, "op"); // '=' + const value = parseExpr(p, 0); + expect(p, "in"); + const body = parseExpr(p, 0); + return { kind: "let", name: name.text, value: value, body: body }; + } + if (t.kind === "if") { + advance(p); + const cond = parseExpr(p, 0); + expect(p, "then"); + const then = parseExpr(p, 0); + expect(p, "else"); + const alt = parseExpr(p, 0); + return { kind: "if", cond: cond, then: then, alt: alt }; + } + advance(p); + return { kind: "num", num: 0 }; +} + +function parseApply(p: Parser): Node { + let head = parseAtom(p); + while (true) { + const t = peek(p); + if (t.kind === "num" || t.kind === "str" || t.kind === "ident" || t.kind === "(") { + const arg = parseAtom(p); + head = { kind: "call", target: head, arg: arg }; + continue; + } + return head; + } +} + +function parseExpr(p: Parser, minPrec: number): Node { + let left = parseApply(p); + while (true) { + const t = peek(p); + if (t.kind !== "op") return left; + const prec = precOf(t.text); + if (prec === 0 || prec < minPrec) return left; + advance(p); + const right = parseExpr(p, prec + 1); + left = { kind: "bin", op: t.text, left: left, right: right }; + } +} + +function parse(src: string): Node { + const p: Parser = { toks: lex(src), pos: 0 }; + return parseExpr(p, 0); +} + +// ---------- evaluator ---------- + +function lookup(env: Env, name: string): Value { + let e: Env | null = env; + while (e !== null) { + const names = e.names; + for (let i = 0; i < names.length; i++) { + if (names[i] === name) return e.vals[i]; + } + e = e.parent; + } + return { kind: "num", num: 0 }; +} + +function asNum(v: Value): number { + if (v.kind === "num") return v.num; + return 0; +} + +function evalNode(n: Node, env: Env): Value { + if (n.kind === "num") return { kind: "num", num: n.num }; + if (n.kind === "str") return { kind: "str", str: n.str }; + if (n.kind === "var") return lookup(env, n.name); + if (n.kind === "fun") return { kind: "clo", param: n.param, body: n.body, env: env }; + if (n.kind === "bin") { + const l = evalNode(n.left, env); + const r = evalNode(n.right, env); + if (n.op === "+" && l.kind === "str") { + const rs = r.kind === "str" ? r.str : "" + asNum(r); + return { kind: "str", str: l.str + rs }; + } + const a = asNum(l); + const b = asNum(r); + if (n.op === "+") return { kind: "num", num: a + b }; + if (n.op === "-") return { kind: "num", num: a - b }; + if (n.op === "*") return { kind: "num", num: a * b }; + if (n.op === "/") return { kind: "num", num: a / b }; + if (n.op === "<") return { kind: "num", num: a < b ? 1 : 0 }; + if (n.op === ">") return { kind: "num", num: a > b ? 1 : 0 }; + if (n.op === "<=") return { kind: "num", num: a <= b ? 1 : 0 }; + return { kind: "num", num: 0 }; + } + if (n.kind === "if") { + const c = evalNode(n.cond, env); + if (asNum(c) !== 0) return evalNode(n.then, env); + return evalNode(n.alt, env); + } + if (n.kind === "let") { + // Recursive let: bind the name first, then evaluate the value inside the + // new scope and patch the slot. A closure defined here captures an + // environment that contains the closure itself — a genuine reference + // cycle, which is the point. + const inner: Env = { names: [n.name], vals: [{ kind: "num", num: 0 }], parent: env }; + const v = evalNode(n.value, inner); + inner.vals[0] = v; + return evalNode(n.body, inner); + } + const fn = evalNode(n.target, env); + const arg = evalNode(n.arg, env); + if (fn.kind === "clo") { + const inner: Env = { names: [fn.param], vals: [arg], parent: fn.env }; + return evalNode(fn.body, inner); + } + return { kind: "num", num: 0 }; +} + +// ---------- driver ---------- + +const FIB = "let fib = fun n if n <= 1 then n else fib (n - 1) + fib (n - 2) in fib 21"; +const SUMLOOP = "let go = fun i if i <= 0 then 0 else i + go (i - 1) in go 250"; +const STRWORK = 'let cat = fun i if i <= 0 then "" else cat (i - 1) + "x" in cat 400'; + +function main(): void { + let checksum = 0; + const progs: string[] = [FIB, SUMLOOP, STRWORK]; + for (let round = 0; round < 400; round++) { + for (let k = 0; k < progs.length; k++) { + const ast = parse(progs[k]); + const env: Env = { names: [], vals: [], parent: null }; + const out = evalNode(ast, env); + if (out.kind === "num") { + checksum = checksum + out.num; + } else if (out.kind === "str") { + checksum = checksum + out.str.length; + } + } + } + console.log(checksum); +} +main(); diff --git a/changelog.d/7758-tls-hot-by-default.md b/changelog.d/7758-tls-hot-by-default.md new file mode 100644 index 0000000000..8bc0890e58 --- /dev/null +++ b/changelog.d/7758-tls-hot-by-default.md @@ -0,0 +1,80 @@ +### `perry_thread_local!`: thread-locals are on the fast path by default, and a gate now says so (#7469) + +On Darwin every `thread_local!` access is an out-of-line call to `_tlv_get_addr` +in libdyld — a real call, not inlined, clobbering caller-saved registers. +`crates/perry-runtime/src/tls_hot.rs` has removed that cost three times, and it +has come back three times: + +| build | workload | `_tlv_get_addr` share | +|---|---|--:| +| after #7565 | `churn_alloc` | 0% | +| later | `churn_alloc` | 8-9% | +| later | `interp` / `retain` | 11% | +| v0.5.1434 | **`asyncpipe`** | **20.5%** | + +**The mechanism never decayed. The coverage policy was the bug.** `HotTls` +carried sixteen hand-wired slots against ~520 `thread_local!` declarations, and +the sixteen were curated against whichever workload was profiled last — the +allocation path. `churn` is covered by construction and reads 0% forever; +`asyncpipe` pays 20.5% through Map/Set registries, buffer brands, descriptor +state and field-lookup tails that were on nobody's list. Adding a slot took +four manual steps including a hand-written test, and *forgetting* them produced +a working slow path rather than a build error. + +#### What changed + +`crate::perry_thread_local!` — same syntax as `thread_local!`, same `with` / +`try_with` at every call site, so converting a declaration converts all of its +uses. The address of the value lands in a generic slot of the same per-thread +cache `hot()` already reaches with an `mrs` plus two loads (#7565), so a read +costs loads instead of a call. There is nothing to wire: no slot to add, no +provider function, no line in `fill`, no line in a test. + +That also removes the hazard the old contract needed a test to catch. The +untyped named slots could hand out a correctly-typed reference to the *wrong* +object if `fill` was mis-wired; here the storage, the resolver and the key's `T` +all come from one declaration, so the mis-pairing cannot be expressed. + +It is also safer on thread teardown than the mechanism it extends. `HotCell` takes `GUARD = needs_drop::() as usize` from the macro: a +`RefCell>` gets a one-element guard array whose `Drop` runs before +the value's and un-publishes this thread's cached address, so a later access +falls back and gets std's "accessed during or after destruction" panic instead +of reading a dropped map. A `Cell` gets a zero-length array, which has no +drop glue at all — no destructor is registered and std's `const`-init fast path +is preserved. The sixteen named fields have no such hook in either direction. + +155 declarations across 51 files — every subsystem the two profiled programs +resolve — now use it. The sixteen named fields are unchanged and stay a closed +set: a fixed offset is one load cheaper than a claimed slot, and the allocation +path is where that matters. + +#### The gates + +Two, because they fail on different things. + +`scripts/check_thread_locals.py` is the structural half: a new raw +`thread_local!` in `perry-runtime` is a build error unless it is recorded in +`scripts/thread_local_cold_allowlist.json` as deliberately cold. The counts are +a ratchet in both directions — a file that *loses* a declaration fails too, +because a stale entry is one nobody has to justify any more. It also fails when +declarations approach `HOT_SLOT_CAPACITY`, since slot exhaustion is correct but +silent. `--self-test` drives all four rejections. + +`scripts/tls_budget_gate.sh` is the outcome half, and its design is about +vacuity. Profiling `churn_alloc` — the benchmark every previous fix was tuned +against — would pass forever while the real cost grew, because churn's +thread-locals are exactly the covered ones: a gate green because its subject +never ran (CLAUDE.md's fourth kind). So the subjects are +`benchmarks/tls-budget/asyncpipe.ts` and `interp.ts`, and +`scripts/tls_budget_check.py` refuses a pass unless the run proves it was live: +`PERRY_TLS_HOT_STATS=1` must report `direct_tsd=1` (otherwise `hot()` is itself +calling `_tlv_get_addr`, the mechanism is inert, and a low share would mean the +program resolved nothing) and `claimed` above a floor no allocation +microbenchmark clears. Its `--self-test` drives seven rejections and runs on +every PR, compiler-free. + +Neither gate is wired into branch protection by this change: a new gate has +never been green, so promoting it immediately would block every open PR. That +is a maintainer action after the first observed green run on `main` — and per +CLAUDE.md's corollary, not optional follow-through. diff --git a/crates/perry-runtime/src/arena/inline.rs b/crates/perry-runtime/src/arena/inline.rs index 9ce8eec0a2..193b9177c1 100644 --- a/crates/perry-runtime/src/arena/inline.rs +++ b/crates/perry-runtime/src/arena/inline.rs @@ -21,22 +21,32 @@ pub struct InlineArenaState { /// is stable for the lifetime of the thread, so caching is safe. /// /// First call on each thread lazy-syncs from the underlying ARENA. +/// +/// #7469: this resolves `INLINE_STATE` and `ARENA` through +/// [`crate::arena::block::hot_inline_state`] / [`crate::arena::block::hot_arena`] +/// rather than `.with()`. Both already *have* named slots in the hot cache — +/// they are two of its sixteen — but this call site was still going through +/// the `LocalKey`, so it paid `_tlv_get_addr` anyway: 5.2% of `interp`'s +/// remaining calls, from a function whose whole job is to be called once per +/// JS function entry. A cache with a covered entry that the caller does not +/// use is the same as no cache. #[no_mangle] pub extern "C" fn js_inline_arena_state() -> *mut InlineArenaState { - INLINE_STATE.with(|s| { - let state = unsafe { &mut *s.get() }; + // SAFETY: `hot_inline_state`/`hot_arena` return this thread's own + // `INLINE_STATE`/`ARENA` storage — the same addresses `.with()` would hand + // out, resolved once per thread instead of once per call. + unsafe { + let state = &mut *super::block::hot_inline_state(); if state.data.is_null() { // Lazy init: copy from underlying ARENA's current block. - ARENA.with(|a| unsafe { - let arena = &*a.get(); - let block = &arena.blocks[arena.current]; - state.data = block.data; - state.offset = block.offset; - state.size = block.size; - }); + let arena = &*super::block::hot_arena(); + let block = &arena.blocks[arena.current]; + state.data = block.data; + state.offset = block.offset; + state.size = block.size; } state as *mut InlineArenaState - }) + } } /// Slow path for inline bump alloc. Called from emitted IR when the diff --git a/crates/perry-runtime/src/array/element_shape.rs b/crates/perry-runtime/src/array/element_shape.rs index 07b75bf89a..03628788c8 100644 --- a/crates/perry-runtime/src/array/element_shape.rs +++ b/crates/perry-runtime/src/array/element_shape.rs @@ -136,7 +136,7 @@ pub(crate) struct ElementShapeProof { pub(crate) epoch: u64, } -thread_local! { +crate::perry_thread_local! { /// Address-keyed element-shape records. `PtrHashMap` for the same reason /// `ARRAY_NAMED_PROPS` uses it (#6386): the key is already a /// well-distributed address and SipHash dominates the probe. diff --git a/crates/perry-runtime/src/array/header.rs b/crates/perry-runtime/src/array/header.rs index 95acf7a923..1144b9303d 100644 --- a/crates/perry-runtime/src/array/header.rs +++ b/crates/perry-runtime/src/array/header.rs @@ -5,7 +5,7 @@ use std::cell::RefCell; use std::collections::HashMap; -thread_local! { +crate::perry_thread_local! { /// Tagged-template `.raw` side-table — maps a cooked-strings array /// pointer to its corresponding raw-strings array pointer. Populated /// by `js_tagged_template_register_raw` at the tagged-call site; read diff --git a/crates/perry-runtime/src/array/iterator.rs b/crates/perry-runtime/src/array/iterator.rs index ba11ca3a4d..7670c2a993 100644 --- a/crates/perry-runtime/src/array/iterator.rs +++ b/crates/perry-runtime/src/array/iterator.rs @@ -552,7 +552,7 @@ extern "C" fn async_from_sync_async_iterator(closure: *const crate::closure::Clo } fn register_async_from_sync_thunks_once() { - thread_local! { + crate::perry_thread_local! { static REGISTERED: std::cell::Cell = const { std::cell::Cell::new(false) }; } REGISTERED.with(|flag| { diff --git a/crates/perry-runtime/src/box.rs b/crates/perry-runtime/src/box.rs index 0a32d90262..f18597423f 100644 --- a/crates/perry-runtime/src/box.rs +++ b/crates/perry-runtime/src/box.rs @@ -30,7 +30,7 @@ pub struct BoolBox { pub value: bool, } -thread_local! { +crate::perry_thread_local! { /// Registry of every active box pointer. GC traces the contained /// JSValue bits so that NaN-boxed heap pointers stored in boxes (e.g. /// the generator state machine's iter object held in `__iter`'s @@ -226,7 +226,7 @@ pub extern "C" fn js_box_get_bits(ptr: *mut Box) -> i64 { } } -thread_local! { +crate::perry_thread_local! { /// #6052: >0 while codegen-emitted Perry-internal materialization reads /// (the `RegisterClassCaptures` decl-site snapshot refresh) are running — /// a dead-zone box then reads as `undefined` (pre-#6044 behavior) instead diff --git a/crates/perry-runtime/src/buffer/detach.rs b/crates/perry-runtime/src/buffer/detach.rs index 38f2e63394..6b50a335f2 100644 --- a/crates/perry-runtime/src/buffer/detach.rs +++ b/crates/perry-runtime/src/buffer/detach.rs @@ -18,7 +18,7 @@ use super::*; use crate::fast_hash::{new_ptr_hash_set, PtrHashSet}; use std::cell::RefCell; -thread_local! { +crate::perry_thread_local! { /// Buffers detached via `transfer`/`transferToFixedLength`/structuredClone /// transfer. A detached buffer also has `length == capacity == 0`, but that /// alone cannot be the probe: `new ArrayBuffer(0)` is empty yet NOT diff --git a/crates/perry-runtime/src/buffer/header.rs b/crates/perry-runtime/src/buffer/header.rs index 39b594affb..e908733664 100644 --- a/crates/perry-runtime/src/buffer/header.rs +++ b/crates/perry-runtime/src/buffer/header.rs @@ -76,7 +76,7 @@ fn notify_crypto_key_death(addr: usize) { pub type CryptoKeyMeta = (u8, u8, u8, bool, u32, u32); -thread_local! { +crate::perry_thread_local! { static BUFFER_REGISTRY: RefCell> = RefCell::new(new_ptr_hash_set()); /// Buffers that were specifically created via `new Uint8Array(...)` — /// formatted as `Uint8Array(N) [ a, b, c ]` instead of ``. diff --git a/crates/perry-runtime/src/buffer/view.rs b/crates/perry-runtime/src/buffer/view.rs index bd46a268b6..ab42ad60c6 100644 --- a/crates/perry-runtime/src/buffer/view.rs +++ b/crates/perry-runtime/src/buffer/view.rs @@ -50,7 +50,7 @@ pub(crate) struct ViewInfo { pub length: u32, } -thread_local! { +crate::perry_thread_local! { /// `view_ptr → ViewInfo`. Lookups during writes are O(1). Address-keyed /// `PtrHashMap` (#6386): both maps are probed on EVERY DataView/typed /// view write via `propagate_written_range_from_receiver`, and SipHash diff --git a/crates/perry-runtime/src/closure/alloc.rs b/crates/perry-runtime/src/closure/alloc.rs index 1ff5dc1a88..6fc236a1cd 100644 --- a/crates/perry-runtime/src/closure/alloc.rs +++ b/crates/perry-runtime/src/closure/alloc.rs @@ -3,7 +3,7 @@ use super::*; use std::cell::RefCell; -thread_local! { +crate::perry_thread_local! { /// Singleton cache keyed by `func_ptr` for non-capturing closures. /// See `js_closure_alloc_singleton` and `scan_singleton_closure_roots_mut`. /// Pointer-keyed; uses `PtrHasher` (Fibonacci-multiplicative) to @@ -302,7 +302,7 @@ const MAX_CAPTURED_CLOSURE_SLOTS: usize = 64; const CAPTURED_MISS_STREAK_DISABLE: u32 = 256; const CAPTURED_DISABLED_SENTINEL: u32 = u32::MAX; -thread_local! { +crate::perry_thread_local! { static CAPTURED_MISS_STREAK: RefCell> = RefCell::new(crate::fast_hash::new_ptr_hash_map()); } diff --git a/crates/perry-runtime/src/closure/dynamic_props.rs b/crates/perry-runtime/src/closure/dynamic_props.rs index aefbc6aabb..69ca4b543a 100644 --- a/crates/perry-runtime/src/closure/dynamic_props.rs +++ b/crates/perry-runtime/src/closure/dynamic_props.rs @@ -643,7 +643,7 @@ pub(crate) fn function_prototype_fallback_target(ptr: usize, prop: &str) -> Opti { return None; } - thread_local! { + crate::perry_thread_local! { static IN_FN_PROTO_FALLBACK: std::cell::Cell = const { std::cell::Cell::new(false) }; } diff --git a/crates/perry-runtime/src/closure/registry.rs b/crates/perry-runtime/src/closure/registry.rs index e9d74f9123..838cd39a31 100644 --- a/crates/perry-runtime/src/closure/registry.rs +++ b/crates/perry-runtime/src/closure/registry.rs @@ -23,7 +23,7 @@ use std::cell::RefCell; // rest-param-bearing closure body in the program; worker threads (issue // #29 `perry/thread`) currently don't see the table because they aren't // supposed to invoke arbitrary user closures across the boundary anyway. -thread_local! { +crate::perry_thread_local! { /// (fixed_arity, kind) — kind describes whether the function has an /// ordinary user rest param, a synthesized `arguments` rest param, or /// both a user rest param plus a hidden raw-arguments slot. @@ -134,7 +134,7 @@ pub enum RestDispatchKind { UserRestAndArguments, } -thread_local! { +crate::perry_thread_local! { /// Last-resolved (func_ptr, strategy) tuple — single-slot direct cache. /// Avoids the per-call HashMap::get + RefCell::borrow when the same /// closure body is invoked back-to-back, which is the steady-state diff --git a/crates/perry-runtime/src/exception.rs b/crates/perry-runtime/src/exception.rs index e87c81181b..da59d2eedb 100644 --- a/crates/perry-runtime/src/exception.rs +++ b/crates/perry-runtime/src/exception.rs @@ -153,7 +153,7 @@ impl ExceptionState { } } -thread_local! { +crate::perry_thread_local! { static EXCEPTION_STATE: std::cell::UnsafeCell = std::cell::UnsafeCell::new(ExceptionState::new()); } diff --git a/crates/perry-runtime/src/gc/roots.rs b/crates/perry-runtime/src/gc/roots.rs index bcb2e07365..0ad8a43af5 100644 --- a/crates/perry-runtime/src/gc/roots.rs +++ b/crates/perry-runtime/src/gc/roots.rs @@ -91,7 +91,7 @@ pub(super) enum RuntimeHandleSlot { HeapWord(u64), } -thread_local! { +crate::perry_thread_local! { pub(super) static ROOT_SCANNERS: RefCell> = RefCell::new(Vec::new()); pub(super) static MUTABLE_ROOT_SCANNERS: RefCell> = const { RefCell::new(Vec::new()) }; pub(super) static FFI_ROOT_SCANNERS: RefCell> = RefCell::new(Vec::new()); diff --git a/crates/perry-runtime/src/lib.rs b/crates/perry-runtime/src/lib.rs index 8dbbdcc979..02ac76111d 100644 --- a/crates/perry-runtime/src/lib.rs +++ b/crates/perry-runtime/src/lib.rs @@ -159,7 +159,8 @@ pub(crate) mod test_support; pub mod text; pub mod timer; /// #7469: one `_tlv_get_addr` for the whole allocation hot path. -pub(crate) mod tls_hot; +#[doc(hidden)] +pub mod tls_hot; pub mod typed_feedback; pub mod typedarray; pub mod typedarray_half; diff --git a/crates/perry-runtime/src/map.rs b/crates/perry-runtime/src/map.rs index 33f8b02914..3913c190e5 100644 --- a/crates/perry-runtime/src/map.rs +++ b/crates/perry-runtime/src/map.rs @@ -14,7 +14,7 @@ use std::ptr; /// Must match value.rs TAG_UNDEFINED const TAG_UNDEFINED: u64 = 0x7FFC_0000_0000_0001; -thread_local! { +crate::perry_thread_local! { static MAP_ITERATOR_ARRAYS: RefCell> = RefCell::new(new_ptr_hash_set()); } @@ -63,7 +63,7 @@ pub(crate) fn test_clear_map_iterator_arrays() { } #[cfg(test)] -thread_local! { +crate::perry_thread_local! { static TEST_FORCE_HELPER_GC: std::cell::Cell = const { std::cell::Cell::new(0) }; } @@ -151,7 +151,7 @@ impl Drop for MapSideAllocation { } } -thread_local! { +crate::perry_thread_local! { static MAP_REGISTRY: RefCell> = RefCell::new(crate::fast_hash::new_ptr_hash_map()); } @@ -326,7 +326,7 @@ fn is_safe_numeric_key(bits: u64) -> bool { // collapse hundreds of keys into bucket 0 (caught by a 2x regression // the first time around). With the avalanche step, even the worst-case // integer-f64 inputs distribute across buckets normally. -thread_local! { +crate::perry_thread_local! { static MAP_INDEX: RefCell< crate::fast_hash::PtrHashMap>, > = RefCell::new(crate::fast_hash::new_ptr_hash_map()); @@ -348,7 +348,7 @@ thread_local! { // Pre-fix `Map.set("key_" + i, …)` over 500k inserts was O(N²) because // each `set` did a linear `find_key_index` to dedup-check; with this // table the dedup probe is O(1) amortized. -thread_local! { +crate::perry_thread_local! { static MAP_STRING_INDEX: RefCell< crate::fast_hash::PtrHashMap>>, > = RefCell::new(crate::fast_hash::new_ptr_hash_map()); @@ -459,7 +459,7 @@ fn is_ptr_index_key(bits: u64) -> bool { // (remembered-set dirty scan, copying field scan, verify/force-evacuate // rewrites), and `map_header_moved_for_gc` migrates the outer key when the // MapHeader itself moves. -thread_local! { +crate::perry_thread_local! { static MAP_PTR_INDEX: RefCell< crate::fast_hash::PtrHashMap>, > = RefCell::new(crate::fast_hash::new_ptr_hash_map()); diff --git a/crates/perry-runtime/src/object/arguments.rs b/crates/perry-runtime/src/object/arguments.rs index de16aa7c1a..2e0583fa5f 100644 --- a/crates/perry-runtime/src/object/arguments.rs +++ b/crates/perry-runtime/src/object/arguments.rs @@ -12,7 +12,7 @@ struct ArgumentsMeta { restricted_callee: bool, } -thread_local! { +crate::perry_thread_local! { static ARGUMENTS_OBJECTS: RefCell> = RefCell::new(crate::fast_hash::new_ptr_hash_map()); } diff --git a/crates/perry-runtime/src/object/async_generator_queue.rs b/crates/perry-runtime/src/object/async_generator_queue.rs index cefc706ef5..dcccaba260 100644 --- a/crates/perry-runtime/src/object/async_generator_queue.rs +++ b/crates/perry-runtime/src/object/async_generator_queue.rs @@ -44,7 +44,7 @@ struct AsyncGeneratorQueueState { queue: VecDeque, } -thread_local! { +crate::perry_thread_local! { static STATES: RefCell> = const { RefCell::new(Vec::new()) }; } @@ -182,7 +182,7 @@ fn set_method(obj: *mut ObjectHeader, name: &[u8], closure: *mut ClosureHeader) /// uninitialized stack slot for `arg` instead of `undefined`. Record arity 1 /// for all three func pointers so the call path pads the missing argument. fn register_wrapper_arities() { - thread_local! { + crate::perry_thread_local! { static REGISTERED: std::cell::Cell = const { std::cell::Cell::new(false) }; } REGISTERED.with(|done| { diff --git a/crates/perry-runtime/src/object/class_constructors.rs b/crates/perry-runtime/src/object/class_constructors.rs index 036f6566df..b8cfd1480b 100644 --- a/crates/perry-runtime/src/object/class_constructors.rs +++ b/crates/perry-runtime/src/object/class_constructors.rs @@ -178,7 +178,7 @@ fn lookup_class_constructor_flags(class_id: u32) -> (bool, bool) { .unwrap_or((false, false)) } -thread_local! { +crate::perry_thread_local! { /// Decl-site snapshots of a function-nested class DECLARATION's captured /// outer locals, keyed by class_id. Filled by the codegen-emitted /// `js_class_register_capture_values` call at the class's source-order diff --git a/crates/perry-runtime/src/object/class_registry/construct.rs b/crates/perry-runtime/src/object/class_registry/construct.rs index d6a962e8c0..f739c070be 100644 --- a/crates/perry-runtime/src/object/class_registry/construct.rs +++ b/crates/perry-runtime/src/object/class_registry/construct.rs @@ -1,7 +1,7 @@ use super::*; use crate::JSValue; -thread_local! { +crate::perry_thread_local! { /// `new.target` for the construction currently on this thread's stack. /// /// **This is a GC root, and must stay one (#7231).** It holds a NaN-boxed diff --git a/crates/perry-runtime/src/object/class_registry/dispatch.rs b/crates/perry-runtime/src/object/class_registry/dispatch.rs index bc4869f687..da1952b412 100644 --- a/crates/perry-runtime/src/object/class_registry/dispatch.rs +++ b/crates/perry-runtime/src/object/class_registry/dispatch.rs @@ -68,7 +68,7 @@ const EMPTY_VTABLE_IC_ENTRY: VTableICEntry = VTableICEntry { has_rest: 0, }; -thread_local! { +crate::perry_thread_local! { // arm64_32 fix: HEAP-allocate (Box) this ~160KB cache instead of inline TLS. // Oversized `#[thread_local]` storage overflows the ILP32 TLS layout and its // writes corrupt adjacent thread-locals. Boxing keeps only a pointer in TLS. diff --git a/crates/perry-runtime/src/object/class_registry/state.rs b/crates/perry-runtime/src/object/class_registry/state.rs index 008dbc61e3..ea34c65e27 100644 --- a/crates/perry-runtime/src/object/class_registry/state.rs +++ b/crates/perry-runtime/src/object/class_registry/state.rs @@ -2,7 +2,7 @@ use super::*; use std::collections::HashMap; use std::sync::RwLock; -thread_local! { +crate::perry_thread_local! { pub(crate) static CLASS_DELETED_KEYS: std::cell::RefCell>> = std::cell::RefCell::new(std::collections::HashMap::new()); } diff --git a/crates/perry-runtime/src/object/collection_proto_thunks.rs b/crates/perry-runtime/src/object/collection_proto_thunks.rs index 7e1761f715..64f9343bcf 100644 --- a/crates/perry-runtime/src/object/collection_proto_thunks.rs +++ b/crates/perry-runtime/src/object/collection_proto_thunks.rs @@ -16,7 +16,7 @@ use super::*; -thread_local! { +crate::perry_thread_local! { static BUILTIN_MAP_SET_VALUE_BITS: std::cell::Cell = const { std::cell::Cell::new(0) }; static BUILTIN_SET_ADD_VALUE_BITS: std::cell::Cell = const { std::cell::Cell::new(0) }; } @@ -31,7 +31,7 @@ pub(crate) fn is_builtin_set_add_value(value: f64) -> bool { fn is_remembered_builtin_collection_method( value: f64, - cell: &'static std::thread::LocalKey>, + cell: &'static crate::tls_hot::HotKey>, ) -> bool { let ptr = normalized_collection_method_ptr(value); ptr != 0 && cell.with(|remembered| remembered.get() == ptr) @@ -41,7 +41,7 @@ fn remember_builtin_collection_method( proto_obj: *mut ObjectHeader, method_name: &str, value: f64, - cell: &'static std::thread::LocalKey>, + cell: &'static crate::tls_hot::HotKey>, ) { let value = installed_collection_method_value(proto_obj, method_name).unwrap_or(value); let ptr = normalized_collection_method_ptr(value); diff --git a/crates/perry-runtime/src/object/exotic_expando.rs b/crates/perry-runtime/src/object/exotic_expando.rs index f000873efb..c6cadaac34 100644 --- a/crates/perry-runtime/src/object/exotic_expando.rs +++ b/crates/perry-runtime/src/object/exotic_expando.rs @@ -101,7 +101,7 @@ pub(crate) fn exotic_expando_kind_of_value(value: f64) -> Option<(usize, ExoticK exotic_expando_kind(addr).map(|kind| (addr, kind)) } -thread_local! { +crate::perry_thread_local! { /// addr -> insertion-ordered (key, nanboxed value bits) pairs (Date/RegExp). static EXOTIC_EXPANDO: RefCell>> = RefCell::new(HashMap::new()); diff --git a/crates/perry-runtime/src/object/field_get_set/accessors.rs b/crates/perry-runtime/src/object/field_get_set/accessors.rs index 170ee97f55..6f57da0a27 100644 --- a/crates/perry-runtime/src/object/field_get_set/accessors.rs +++ b/crates/perry-runtime/src/object/field_get_set/accessors.rs @@ -114,7 +114,7 @@ pub(crate) unsafe fn own_data_field_by_name( None } -thread_local! { +crate::perry_thread_local! { static OBJECT_PROTOTYPE_LOOKUP_DEPTH: std::cell::Cell = const { std::cell::Cell::new(0) }; } @@ -224,7 +224,7 @@ pub(crate) unsafe fn ordinary_object_prototype_property_value( default_object_prototype_property_value(obj as usize, key) } -thread_local! { +crate::perry_thread_local! { /// Receiver to bind when an accessor getter is reached by walking a /// prototype chain. `js_object_get_field_by_name(proto, key)` re-derives the /// accessor receiver from its `obj` argument — which is the PROTOTYPE during diff --git a/crates/perry-runtime/src/object/field_get_set/field_ops.rs b/crates/perry-runtime/src/object/field_get_set/field_ops.rs index cc5b647048..ad18e9ea3c 100644 --- a/crates/perry-runtime/src/object/field_get_set/field_ops.rs +++ b/crates/perry-runtime/src/object/field_get_set/field_ops.rs @@ -34,7 +34,7 @@ use super::*; const WARN_NULL_PTR_LOG_LIMIT: u64 = 64; const WARN_NULL_PTR_ABORT_LIMIT: u64 = 100_000; -thread_local! { +crate::perry_thread_local! { static WARN_NULL_PTR_STATE: std::cell::Cell = const { std::cell::Cell::new(WarnNullPtrState { total_count: 0, diff --git a/crates/perry-runtime/src/object/global_fetch.rs b/crates/perry-runtime/src/object/global_fetch.rs index 323872e551..a4284fc977 100644 --- a/crates/perry-runtime/src/object/global_fetch.rs +++ b/crates/perry-runtime/src/object/global_fetch.rs @@ -8,7 +8,7 @@ use std::cell::Cell; use std::ptr::null_mut; use std::sync::atomic::{AtomicPtr, Ordering}; -thread_local! { +crate::perry_thread_local! { /// The `signal` from the in-progress `fetch(url, { signal })` call, stashed /// so the stdlib `js_fetch_with_options` (whose 4-arg ABI predates /// AbortSignal support) can pick it up at entry without an ABI change. diff --git a/crates/perry-runtime/src/object/global_this/fetch_globals.rs b/crates/perry-runtime/src/object/global_this/fetch_globals.rs index e342ab31f2..3b31e7d9e0 100644 --- a/crates/perry-runtime/src/object/global_this/fetch_globals.rs +++ b/crates/perry-runtime/src/object/global_this/fetch_globals.rs @@ -1,7 +1,7 @@ use super::super::*; use super::*; -thread_local! { +crate::perry_thread_local! { /// This thread's `globalThis`. The realm global is allocated in a *per-thread* /// arena, but `GLOBAL_THIS_PTR` (the GC-root slot) is a process-global static. /// A pointer published there by another, now-finished thread (the unit-test @@ -13,7 +13,7 @@ thread_local! { static THREAD_GLOBAL_THIS: std::cell::Cell = const { std::cell::Cell::new(0) }; } -thread_local! { +crate::perry_thread_local! { /// Module top-level `this` (Node-CJS `module.exports` stand-in) — a /// lazily-allocated plain object distinct from `globalThis`. See /// `Expr::ModuleTopThis`. diff --git a/crates/perry-runtime/src/object/global_this/populate.rs b/crates/perry-runtime/src/object/global_this/populate.rs index b2184b5c4a..c93b4ba2bd 100644 --- a/crates/perry-runtime/src/object/global_this/populate.rs +++ b/crates/perry-runtime/src/object/global_this/populate.rs @@ -848,7 +848,7 @@ fn alias_number_static_to_global_function(singleton_at_entry: *mut ObjectHeader, ); } -thread_local! { +crate::perry_thread_local! { /// Raw address of THIS thread's `Error` constructor closure, captured at /// install. Read by `error::error_prepare_stack_trace_override` so /// `captureStackTrace` / `error.stack` can honor a user-set diff --git a/crates/perry-runtime/src/object/handle_expando.rs b/crates/perry-runtime/src/object/handle_expando.rs index edcd06c2a4..731b36bbba 100644 --- a/crates/perry-runtime/src/object/handle_expando.rs +++ b/crates/perry-runtime/src/object/handle_expando.rs @@ -50,7 +50,7 @@ use std::collections::HashMap; // keys come back in INSERTION order — `Object.keys(handle)` / `{...handle}` are // ordered in JS, and handles carry a handful of expandos at most, so the linear // scan is cheaper than hashing. -thread_local! { +crate::perry_thread_local! { static HANDLE_EXPANDO_PROPS: RefCell>> = RefCell::new(HashMap::new()); } diff --git a/crates/perry-runtime/src/object/instanceof.rs b/crates/perry-runtime/src/object/instanceof.rs index 36d875b265..e9c6377b7a 100644 --- a/crates/perry-runtime/src/object/instanceof.rs +++ b/crates/perry-runtime/src/object/instanceof.rs @@ -747,7 +747,7 @@ fn ordinary_has_instance(constructor: f64, value: f64) -> bool { result.to_bits() == crate::value::TAG_TRUE } -thread_local! { +crate::perry_thread_local! { /// When set, `js_instanceof_dynamic` returns `false` instead of throwing on /// an unresolved / non-callable right-hand side. Used by /// `OrdinaryHasInstance` (#3662), whose spec returns `false` there rather diff --git a/crates/perry-runtime/src/object/mod.rs b/crates/perry-runtime/src/object/mod.rs index bedd339485..a2c1fee463 100644 --- a/crates/perry-runtime/src/object/mod.rs +++ b/crates/perry-runtime/src/object/mod.rs @@ -266,7 +266,7 @@ pub(crate) static SESSION_STORAGE_PTR: AtomicI64 = AtomicI64::new(0); // // This handles cases like Object.assign() adding many fields to an object // that was allocated with only 8 slots (e.g., @noble/curves Fp field with 21 properties). -thread_local! { +crate::perry_thread_local! { static CLASS_PROTOTYPE_METHOD_VALUES: RefCell> = RefCell::new(HashMap::new()); } @@ -417,7 +417,7 @@ fn keys_index_insert( // Recursion depth guard for js_native_call_method to prevent stack overflow // from circular module dependencies during initialization. -thread_local! { +crate::perry_thread_local! { static CALL_METHOD_DEPTH: Cell = const { Cell::new(0) }; } const MAX_CALL_METHOD_DEPTH: u32 = 512; @@ -553,7 +553,7 @@ pub(crate) struct ShapeCacheEntry { keys_array: *mut ArrayHeader, } -thread_local! { +crate::perry_thread_local! { /// Issue #618-followup / drizzle SQL.Aliased: dynamic properties added /// via the IIFE pattern `((SQL2) => { SQL2.Aliased = Aliased; })(SQL)` /// to imported classes (which Perry stores as INT32-tagged class ids). diff --git a/crates/perry-runtime/src/object/native_module.rs b/crates/perry-runtime/src/object/native_module.rs index f614a9f0bc..83e6937853 100644 --- a/crates/perry-runtime/src/object/native_module.rs +++ b/crates/perry-runtime/src/object/native_module.rs @@ -46,7 +46,7 @@ pub(crate) use namespace_builders::{ }; pub(crate) use web_locks::{worker_threads_locks_value, WebLocksState}; -thread_local! { +crate::perry_thread_local! { pub(crate) static NATIVE_CALLABLE_EXPORTS: RefCell> = RefCell::new(HashMap::new()); pub(crate) static NATIVE_MODULE_ACCESSOR_EXPORTS: RefCell> = diff --git a/crates/perry-runtime/src/object/native_module_stream.rs b/crates/perry-runtime/src/object/native_module_stream.rs index 55d95e0e50..98532f8b1e 100644 --- a/crates/perry-runtime/src/object/native_module_stream.rs +++ b/crates/perry-runtime/src/object/native_module_stream.rs @@ -2,7 +2,7 @@ use super::*; -thread_local! { +crate::perry_thread_local! { static STREAM_EVENT_EMITTER_PROTOTYPES: RefCell> = const { RefCell::new(Vec::new()) }; } diff --git a/crates/perry-runtime/src/object/native_this_alias.rs b/crates/perry-runtime/src/object/native_this_alias.rs index db96bda318..053ca31eb0 100644 --- a/crates/perry-runtime/src/object/native_this_alias.rs +++ b/crates/perry-runtime/src/object/native_this_alias.rs @@ -63,7 +63,7 @@ fn object_addr_of(value: f64) -> usize { } } -thread_local! { +crate::perry_thread_local! { static ALIAS_ACTIVE: Cell = const { Cell::new(false) }; static ALIASES: RefCell> = const { RefCell::new(Vec::new()) }; } diff --git a/crates/perry-runtime/src/object/prop_plan.rs b/crates/perry-runtime/src/object/prop_plan.rs index 00d56da0f3..6a098ea53c 100644 --- a/crates/perry-runtime/src/object/prop_plan.rs +++ b/crates/perry-runtime/src/object/prop_plan.rs @@ -67,7 +67,7 @@ struct PlanEntry { const PLAN_CACHE_SIZE: usize = 4096; const PLAN_CACHE_MASK: usize = PLAN_CACHE_SIZE - 1; -thread_local! { +crate::perry_thread_local! { // Heap-allocate the table (~112KB) — oversized inline TLS overflows the // ILP32 TLS layout on arm64_32 (same fix as string/intern.rs). static STORE_PLAN_CACHE: std::cell::UnsafeCell> = @@ -181,7 +181,7 @@ struct ReadPlanEntry { const READ_PLAN_SIZE: usize = 8192; const READ_PLAN_MASK: usize = READ_PLAN_SIZE - 1; -thread_local! { +crate::perry_thread_local! { // Heap-allocated for the same arm64_32 TLS-size reason as the store table. static READ_PLAN_CACHE: std::cell::UnsafeCell> = std::cell::UnsafeCell::new( diff --git a/crates/perry-runtime/src/object/prototype_chain.rs b/crates/perry-runtime/src/object/prototype_chain.rs index c4d4dee4b7..f3924fce38 100644 --- a/crates/perry-runtime/src/object/prototype_chain.rs +++ b/crates/perry-runtime/src/object/prototype_chain.rs @@ -44,7 +44,7 @@ pub(crate) fn array_static_proto_recorded() -> bool { const TAG_NULL: u64 = 0x7FFC_0000_0000_0002; static OBJECT_PROTOTYPES: OnceLock>> = OnceLock::new(); -thread_local! { +crate::perry_thread_local! { /// Owners currently walking a recorded prototype chain. Although /// `Object.setPrototypeOf` normally rejects cycles, residual/native owners /// and custom-construction links can still expose a malformed chain. Keep diff --git a/crates/perry-runtime/src/object/this_binding.rs b/crates/perry-runtime/src/object/this_binding.rs index 7f854a9735..6f68a0a882 100644 --- a/crates/perry-runtime/src/object/this_binding.rs +++ b/crates/perry-runtime/src/object/this_binding.rs @@ -23,7 +23,7 @@ use std::cell::Cell; // // Defaults to `TAG_UNDEFINED`. JS spec says top-level `this` is undefined // in strict mode, which matches. -thread_local! { +crate::perry_thread_local! { pub(crate) static IMPLICIT_THIS: Cell = const { Cell::new(crate::value::TAG_UNDEFINED) }; pub(crate) static NEW_TARGET: Cell = const { Cell::new(crate::value::TAG_UNDEFINED) }; // One-shot receiver override for STATIC method bodies. A compiled static diff --git a/crates/perry-runtime/src/promise/async_step.rs b/crates/perry-runtime/src/promise/async_step.rs index 38ea2d5f9a..a37f5d570a 100644 --- a/crates/perry-runtime/src/promise/async_step.rs +++ b/crates/perry-runtime/src/promise/async_step.rs @@ -14,7 +14,7 @@ use super::*; // evaporates" signature. Find it by diffing the unmatched-await backtraces of // a working run against a hanging one; the SUSPEND line carries the site. // A no-op (one cached bool check) when the env var is unset. -thread_local! { +crate::perry_thread_local! { static TRACE_ASYNC_ON: std::cell::Cell = const { std::cell::Cell::new(-1) }; static TRACE_ASYNC_SEQ: std::cell::Cell = const { std::cell::Cell::new(0) }; static TRACE_ASYNC_AWAITED: std::cell::RefCell> = @@ -853,7 +853,7 @@ static KEEP_JS_ASYNC_GENERATOR_RESUME: extern "C" fn(f64, f64, f64) -> f64 = // return the cached thunks; otherwise we allocate. The thunks are // GC-rooted via `ASYNC_STEP_THUNK_CACHE_SCANNER` so they survive // collection until evicted by a different step closure. -thread_local! { +crate::perry_thread_local! { pub(super) static LAST_ASYNC_STEP_THUNKS: std::cell::Cell<(usize, *mut crate::closure::ClosureHeader, *mut crate::closure::ClosureHeader)> = const { std::cell::Cell::new((0, std::ptr::null_mut(), std::ptr::null_mut())) }; } diff --git a/crates/perry-runtime/src/promise/combinators.rs b/crates/perry-runtime/src/promise/combinators.rs index f9efc3972e..7c5456e5d5 100644 --- a/crates/perry-runtime/src/promise/combinators.rs +++ b/crates/perry-runtime/src/promise/combinators.rs @@ -20,7 +20,7 @@ pub(crate) struct PromiseAllState { pub index: u32, } -thread_local! { +crate::perry_thread_local! { /// Keyed by input-promise address. See `keyed_table.rs`: a dense `Vec` is /// still the GC scanners' traversal/rewrite surface, with an O(1) key index /// layered on top (#6084 item 2 — this used to be a raw `Vec` that every @@ -531,7 +531,7 @@ static KEEP_PROMISE_ALL_SETTLED_ITERABLE: extern "C" fn(f64) -> *mut Promise = static KEEP_PROMISE_ANY_ITERABLE: extern "C" fn(f64) -> *mut Promise = js_promise_any_iterable; // Queue for scheduled promise resolutions -thread_local! { +crate::perry_thread_local! { pub(in crate::promise) static SCHEDULED_RESOLVES: RefCell> = const { RefCell::new(Vec::new()) }; } @@ -658,7 +658,7 @@ fn take_already_resolved(guard: *mut crate::array::ArrayHeader) -> bool { /// (observed as the denormal `5e-324`, i.e. bits = 1), corrupting the /// resolution value. (test262 exception-after-resolve-in-{executor,thenable-job}.) pub(super) fn ensure_native_resolving_arity_registered() { - thread_local! { + crate::perry_thread_local! { static DONE: std::cell::Cell = const { std::cell::Cell::new(false) }; } DONE.with(|d| { diff --git a/crates/perry-runtime/src/promise/microtasks.rs b/crates/perry-runtime/src/promise/microtasks.rs index 5511fe6033..7c33815842 100644 --- a/crates/perry-runtime/src/promise/microtasks.rs +++ b/crates/perry-runtime/src/promise/microtasks.rs @@ -4,7 +4,7 @@ use super::*; -thread_local! { +crate::perry_thread_local! { /// Promise currently being dispatched by the microtask runner after its /// task has been popped from TASK_QUEUE. While user callbacks run this is /// the mutable root that lets copied-minor rewrite the promise pointer diff --git a/crates/perry-runtime/src/promise/mod.rs b/crates/perry-runtime/src/promise/mod.rs index 4177c23759..6be26a4861 100644 --- a/crates/perry-runtime/src/promise/mod.rs +++ b/crates/perry-runtime/src/promise/mod.rs @@ -287,7 +287,7 @@ pub(crate) fn mt_profile_register() { // generators (async functions rewritten via async→generator). User- // visible generators (`function*`) still allocate real `{value, done}` // objects so `for...of` and external consumers see the spec shape. -thread_local! { +crate::perry_thread_local! { static ITER_RESULT_VALUE: std::cell::Cell = const { std::cell::Cell::new(0.0) }; static ITER_RESULT_VALUE_I32: std::cell::Cell = const { std::cell::Cell::new(0) }; static ITER_RESULT_VALUE_I1: std::cell::Cell = const { std::cell::Cell::new(false) }; @@ -574,7 +574,7 @@ pub(crate) enum Task { // their continuations in source order (1 first, then 2). Using a // `Vec` with `.pop()` produces LIFO ordering, breaking every test // that prints inside multiple parallel promise chains. -thread_local! { +crate::perry_thread_local! { pub(crate) static TASK_QUEUE: RefCell> = const { RefCell::new(std::collections::VecDeque::new()) }; diff --git a/crates/perry-runtime/src/promise/reactions.rs b/crates/perry-runtime/src/promise/reactions.rs index d33fb05129..62fe9f502a 100644 --- a/crates/perry-runtime/src/promise/reactions.rs +++ b/crates/perry-runtime/src/promise/reactions.rs @@ -16,7 +16,7 @@ pub(super) struct PromiseSettleListener { pub(super) context: AsyncContextSnapshot, } -thread_local! { +crate::perry_thread_local! { /// Keyed by pending-promise address — see `keyed_table.rs` (#6084 item 2: /// this used to be a raw `Vec` that every settlement scanned end to end). pub(super) static PROMISE_SETTLE_LISTENERS: RefCell> = @@ -159,7 +159,7 @@ pub(super) struct OverflowReaction { pub(super) context: AsyncContextSnapshot, } -thread_local! { +crate::perry_thread_local! { /// Keyed by pending-promise address — see `keyed_table.rs` (#6084 item 2: /// this used to be a raw `Vec` that every settlement scanned end to end). pub(super) static PROMISE_OVERFLOW_REACTIONS: RefCell> = diff --git a/crates/perry-runtime/src/promise/rejection.rs b/crates/perry-runtime/src/promise/rejection.rs index e3b89a6afe..7a1bdb7090 100644 --- a/crates/perry-runtime/src/promise/rejection.rs +++ b/crates/perry-runtime/src/promise/rejection.rs @@ -37,7 +37,7 @@ use super::*; -thread_local! { +crate::perry_thread_local! { static REJECTIONS: RefCell = RefCell::new(RejectionTracker::default()); /// Re-entrancy guard: a listener invoked from a checkpoint can run /// arbitrary JS (including code that drains microtasks); it must not diff --git a/crates/perry-runtime/src/promise/spec_combinators.rs b/crates/perry-runtime/src/promise/spec_combinators.rs index dd2f95f08f..adad9a5bde 100644 --- a/crates/perry-runtime/src/promise/spec_combinators.rs +++ b/crates/perry-runtime/src/promise/spec_combinators.rs @@ -84,7 +84,7 @@ pub(super) struct Capability { // missing slots with `undefined` (rather than reading uninitialised registers). // --------------------------------------------------------------------------- -thread_local! { +crate::perry_thread_local! { static ARITY_REGISTERED: std::cell::Cell = const { std::cell::Cell::new(false) }; } diff --git a/crates/perry-runtime/src/promise/then.rs b/crates/perry-runtime/src/promise/then.rs index 0367e35d21..a37073b96c 100644 --- a/crates/perry-runtime/src/promise/then.rs +++ b/crates/perry-runtime/src/promise/then.rs @@ -1361,7 +1361,7 @@ extern "C" fn spec_catch_finally_fn( } fn ensure_spec_finally_arities_registered() { - thread_local! { + crate::perry_thread_local! { static DONE: std::cell::Cell = const { std::cell::Cell::new(false) }; } DONE.with(|d| { diff --git a/crates/perry-runtime/src/regex.rs b/crates/perry-runtime/src/regex.rs index 0444a3070d..519d686a35 100644 --- a/crates/perry-runtime/src/regex.rs +++ b/crates/perry-runtime/src/regex.rs @@ -94,7 +94,7 @@ pub use exec::js_regexp_exec; #[cfg(feature = "regex-engine")] pub use match_string::{js_string_match, js_string_match_value, js_string_search_value}; -thread_local! { +crate::perry_thread_local! { /// Last exec result metadata: (index, groups_object_ptr) /// Stored per-thread so that `m.index` and `m.groups` can retrieve them /// after the exec call. @@ -218,7 +218,7 @@ pub(crate) unsafe fn regex_gc_slot_ptrs(re: *mut RegExpHeader) -> (*mut u64, usi } #[cfg(feature = "regex-engine")] -thread_local! { +crate::perry_thread_local! { /// Cache of compiled regex objects, keyed by (pattern, flags). static REGEX_CACHE: RefCell>> = RefCell::new(HashMap::new()); /// Fancy-regex fallback cache for patterns with lookbehind/lookahead. diff --git a/crates/perry-runtime/src/set.rs b/crates/perry-runtime/src/set.rs index e7b336f3f6..316ac05c78 100644 --- a/crates/perry-runtime/src/set.rs +++ b/crates/perry-runtime/src/set.rs @@ -11,7 +11,7 @@ use std::cell::RefCell; use std::hash::{Hash, Hasher}; use std::ptr; -thread_local! { +crate::perry_thread_local! { static SET_ITERATOR_ARRAYS: RefCell> = RefCell::new(new_ptr_hash_set()); } @@ -60,7 +60,7 @@ pub(crate) fn test_clear_set_iterator_arrays() { } #[cfg(test)] -thread_local! { +crate::perry_thread_local! { static TEST_FORCE_HELPER_GC: std::cell::Cell = const { std::cell::Cell::new(false) }; } @@ -140,7 +140,7 @@ impl Drop for SetSideAllocation { } } -thread_local! { +crate::perry_thread_local! { static SET_REGISTRY: RefCell> = RefCell::new(crate::fast_hash::new_ptr_hash_map()); } @@ -190,7 +190,7 @@ impl Eq for JSValueKey {} // avalanche step handles both cleanly. Same rationale as MAP_INDEX // (commit 39e253cd) — the perry-runtime registries don't need // SipHash's DoS-resistance for keys that never come from external input. -thread_local! { +crate::perry_thread_local! { static SET_INDEX: RefCell< crate::fast_hash::PtrHashMap>, > = RefCell::new(crate::fast_hash::new_ptr_hash_map()); diff --git a/crates/perry-runtime/src/state.rs b/crates/perry-runtime/src/state.rs index 35dde1e4bb..332d4ea0e9 100644 --- a/crates/perry-runtime/src/state.rs +++ b/crates/perry-runtime/src/state.rs @@ -56,12 +56,20 @@ impl RuntimeState { } } -thread_local! { +crate::perry_thread_local! { /// Fast-path pointer to this thread's state. `Cell<*mut _>` has no drop /// glue, so this TLS slot never registers a destructor — `with` on it /// compiles down to the raw TLS address computation + load, and it /// remains accessible from other TLS destructors during thread /// teardown. + /// + /// #7469: "the raw TLS address computation" is an out-of-line + /// `_tlv_get_addr` call on Darwin, and [`state`] is on the miss path of + /// every inline-cache property read. It was **83% of `interp`'s remaining + /// `_tlv_get_addr` calls** — the largest single site left after the + /// registries were converted. This module is itself a consolidation + /// (#6759 folded N side tables into one state behind one TLS pointer); + /// what it could not do is make that one pointer free to reach. static STATE_PTR: Cell<*mut RuntimeState> = const { Cell::new(std::ptr::null_mut()) }; /// Owns the allocation behind [`STATE_PTR`]; its destructor frees the /// state at thread exit (and nulls the fast-path pointer first, so a diff --git a/crates/perry-runtime/src/string/format.rs b/crates/perry-runtime/src/string/format.rs index fe0a8ef083..2e46dc83e4 100644 --- a/crates/perry-runtime/src/string/format.rs +++ b/crates/perry-runtime/src/string/format.rs @@ -12,7 +12,7 @@ use super::*; /// across threads would hand back arena pointers that are invalid in /// the caller's address space (use-after-free / cross-arena UB). const SMALL_INT_CACHE_SIZE: usize = 256; -thread_local! { +crate::perry_thread_local! { static SMALL_INT_CACHE: std::cell::UnsafeCell<[*mut StringHeader; SMALL_INT_CACHE_SIZE]> = const { std::cell::UnsafeCell::new([std::ptr::null_mut(); SMALL_INT_CACHE_SIZE]) }; } diff --git a/crates/perry-runtime/src/string/intern.rs b/crates/perry-runtime/src/string/intern.rs index 7b9b8a29dd..e0f07b0ba8 100644 --- a/crates/perry-runtime/src/string/intern.rs +++ b/crates/perry-runtime/src/string/intern.rs @@ -25,7 +25,7 @@ pub(crate) const INTERN_MAX_BYTE_LEN: u32 = 64; // read from worker B. The previous design used a single process-wide // `static mut`, which both raced under concurrent allocation and risked // handing back foreign-arena pointers. -thread_local! { +crate::perry_thread_local! { // arm64_32 fix: HEAP-allocate this table instead of inline TLS. // Oversized `#[thread_local]` storage overflows the ILP32 TLS layout and its // writes corrupt adjacent thread-locals. Boxing keeps only a pointer in TLS. diff --git a/crates/perry-runtime/src/symbol.rs b/crates/perry-runtime/src/symbol.rs index 0ab8d4248d..98fd844af4 100644 --- a/crates/perry-runtime/src/symbol.rs +++ b/crates/perry-runtime/src/symbol.rs @@ -137,7 +137,7 @@ pub(crate) fn registered_symbol_description(sym_ptr: usize) -> Option *mut u8` next to the `thread_local!` -//! that owns the storage, returning `KEY.with(|k| k as *const _ as *mut u8)`. -//! 3. Wire it in [`fill`]. -//! 4. Add the pair to `tls_hot::tests::cached_addresses_match_thread_locals`. +//! The sixteen named fields below are a **closed set**. They are the +//! allocation path, they have fixed offsets, and they are kept because a fixed +//! offset is one load cheaper than a claimed slot on the hottest path in the +//! runtime. Nothing else belongs there. //! -//! Step 4 is the load-bearing one: the slots are untyped (`*mut u8`) so the -//! owning module can keep its storage type private, which means a mis-wired +//! Adding one used to take four manual steps — slot, `…_hot_addr()` provider, +//! a line in [`fill`], and a line in +//! `tests::cached_addresses_match_thread_locals` — and step four was +//! load-bearing, because the slots are untyped (`*mut u8`) so a mis-wired //! `fill` would hand out a correctly-typed reference to the *wrong* object. -//! The test compares each cached address against the `.with()` address it is -//! supposed to mirror, so a mis-wire is a red build rather than a silent -//! cross-cast. +//! +//! That contract cannot scale to ~520 declarations, and it gets the default +//! backwards: forgetting it produces a **working slow path**, not a build +//! error. Which is why this cost was fixed three times and came back three +//! times — measured 0% of `churn_alloc` after #7565 (`churn` is covered by +//! construction), 8-9% later, 11% on `interp`/`retain`, and 20.5% of +//! `asyncpipe`, whose Map/Set registries, buffer brands and descriptor state +//! were on nobody's list. +//! +//! [`crate::perry_thread_local`] is the default now: same syntax as +//! `thread_local!`, same `with`/`try_with` at every call site, and the address +//! lands in a generic slot of this same cache with **nothing to wire**. The +//! declaration generates its own storage, its own resolver and its own typed +//! key, so the cross-cast hazard above cannot be expressed — and it installs a +//! teardown guard exactly when the value has a destructor, which the named +//! fields do not have at all. `scripts/check_thread_locals.py` makes a new raw +//! `thread_local!` a build error unless it is recorded as deliberately cold, +//! and `scripts/tls_budget_gate.sh` measures the outcome on two programs whose +//! paths are deliberately *not* among these sixteen. //! //! # Lifetime //! @@ -83,7 +100,28 @@ //! every thread falls back to `_tlv_get_addr`, permanently and silently //! correctly. It cannot degrade into reading a wrong address. -use std::cell::UnsafeCell; +use std::cell::{Cell, UnsafeCell}; + +/// How many generic [`HotKey`] slots one thread's cache can hold. +/// +/// One `*mut u8` each, in `__thread_bss`, so the cost is address space rather +/// than image size. `perry-runtime` declares ~520 `thread_local!`s in total, so +/// this leaves headroom for every one of them plus `perry-stdlib`'s. +/// +/// Overflow is *correct* — a declaration that cannot get a slot simply falls +/// back to the plain `thread_local!` path forever — but it is **silent**, which +/// is precisely the failure mode this file exists to abolish. So two things +/// watch it: `scripts/check_thread_locals.py` fails the build when the +/// declaration count approaches this ceiling, and `claimed_slots` lets the +/// runtime budget gate reject a run that reached it. +pub const HOT_SLOT_CAPACITY: usize = 768; + +/// A [`SlotId`] that has never been claimed. +const SLOT_UNASSIGNED: u32 = u32::MAX; +/// A [`SlotId`] that asked for a slot after the last one was handed out. Both +/// sentinels are `>= HOT_SLOT_CAPACITY`, so the one bound check on the hot path +/// rejects them together. +const SLOT_OVERFLOW: u32 = u32::MAX - 1; /// Cached addresses of the per-thread state on the allocation hot path. /// @@ -115,9 +153,31 @@ pub(crate) struct HotTls { pub(crate) learned_inline_fields: *mut u8, // gc/roots/temp_roots.rs pub(crate) temp_roots: *mut u8, + /// Generic slots, one per [`crate::perry_thread_local`] declaration that + /// this thread has resolved at least once. Last, so the named fields above + /// keep their small fixed offsets. + slots: [Cell<*mut u8>; HOT_SLOT_CAPACITY], } impl HotTls { + /// Read a claimed slot. `idx` must have passed the `< HOT_SLOT_CAPACITY` + /// test that both sentinels fail. + #[inline(always)] + fn slot(&self, idx: u32) -> *mut u8 { + debug_assert!((idx as usize) < HOT_SLOT_CAPACITY); + // SAFETY: the caller checked the bound; this elides the panic path from + // every hot read, which is the whole point of the check being a single + // unsigned compare against a constant. + unsafe { self.slots.get_unchecked(idx as usize).get() } + } + + #[inline(always)] + fn set_slot(&self, idx: u32, value: *mut u8) { + debug_assert!((idx as usize) < HOT_SLOT_CAPACITY); + // SAFETY: as `slot` above. + unsafe { self.slots.get_unchecked(idx as usize).set(value) } + } + const EMPTY: Self = Self { arena: std::ptr::null_mut(), inline_state: std::ptr::null_mut(), @@ -135,6 +195,7 @@ impl HotTls { shape_install_memo: std::ptr::null_mut(), learned_inline_fields: std::ptr::null_mut(), temp_roots: std::ptr::null_mut(), + slots: [const { Cell::new(std::ptr::null_mut()) }; HOT_SLOT_CAPACITY], }; } @@ -378,6 +439,449 @@ pub(crate) fn hot() -> &'static HotTls { hot_via_tls() } +// --------------------------------------------------------------------------- +// Generic slots: the same collapse, without a list anyone has to maintain. +// --------------------------------------------------------------------------- + +/// The process-wide slot index for one [`crate::perry_thread_local`] +/// declaration. +/// +/// Claimed once, on the first thread that resolves the declaration, and stable +/// for the life of the process — so every thread finds the same declaration at +/// the same index in its own cache. +pub struct SlotId(std::sync::atomic::AtomicU32); + +impl SlotId { + pub const fn new() -> Self { + Self(std::sync::atomic::AtomicU32::new(SLOT_UNASSIGNED)) + } + + /// The claimed index, or a sentinel `>= HOT_SLOT_CAPACITY`. + /// + /// Relaxed is sufficient: the index is a pure allocation decision, and the + /// *pointer* it selects is per-thread and published by that same thread. + #[inline(always)] + fn raw(&self) -> u32 { + self.0.load(std::sync::atomic::Ordering::Relaxed) + } + + /// Claim this declaration's index, once per process. + /// + /// Under a mutex rather than a bare `fetch_add` because a lost race would + /// *leak* the index it lost with: eight `parallelMap` workers first-touching + /// the same declaration together would burn eight slots for one declaration, + /// and `HOT_SLOT_CAPACITY` is sized for declarations, not for declarations + /// times threads. + #[cold] + #[inline(never)] + fn claim(&self) -> u32 { + use std::sync::atomic::Ordering; + maybe_install_stats_hook(); + let mut next = match CLAIM_LOCK.lock() { + Ok(next) => next, + Err(poisoned) => poisoned.into_inner(), + }; + let current = self.0.load(Ordering::Relaxed); + if current != SLOT_UNASSIGNED { + return current; + } + let idx = if (*next as usize) < HOT_SLOT_CAPACITY { + let idx = *next; + *next += 1; + idx + } else { + SLOT_OVERFLOW + }; + self.0.store(idx, Ordering::Relaxed); + idx + } +} + +impl Default for SlotId { + fn default() -> Self { + Self::new() + } +} + +/// The next index [`SlotId::claim`] will hand out. Also the count of +/// declarations claimed so far, which is what +/// [`claimed_slots`] reports and what the capacity test asserts against. +static CLAIM_LOCK: std::sync::Mutex = std::sync::Mutex::new(0); + +/// How many declarations have claimed a slot in this process. +/// +/// Instrumentation for the capacity assertion: overflow is silent by design +/// (the declaration keeps working, slowly), so something has to be able to see +/// how close the process is to the ceiling. +pub fn claimed_slots() -> u32 { + match CLAIM_LOCK.lock() { + Ok(next) => *next, + Err(poisoned) => *poisoned.into_inner(), + } +} + +/// How many slots *this thread* has populated. +pub fn published_slots() -> usize { + hot().slots.iter().filter(|s| !s.get().is_null()).count() +} + +/// `PERRY_TLS_HOT_STATS=1` — print, at process exit, what this mechanism +/// actually did. +/// +/// This exists so a budget gate can assert its subject was LIVE rather than +/// merely quiet. `_tlv_get_addr` reading 0% is the *same observation* whether +/// the cache carried the program's thread-locals or the program simply never +/// resolved one, and #7469's history is that the second case shipped as a pass +/// three times. The line reports: +/// +/// * `claimed` — declarations that took a slot process-wide. A program that +/// exercises paths outside the sixteen named fields drives this well past +/// zero; a program that does not, does not. +/// * `published` — slots this thread actually filled. +/// * `direct_tsd` — whether `hot()` is the `mrs`-plus-two-loads path. `0` +/// means the self-check rejected direct addressing and every access is +/// paying `_tlv_get_addr` again, i.e. the whole mechanism is inert. +fn maybe_install_stats_hook() { + static INSTALLED: std::sync::Once = std::sync::Once::new(); + INSTALLED.call_once(|| { + if !matches!( + std::env::var("PERRY_TLS_HOT_STATS").as_deref(), + Ok("1") | Ok("on") | Ok("true") + ) { + return; + } + extern "C" fn report() { + #[cfg(all( + target_vendor = "apple", + target_arch = "aarch64", + target_pointer_width = "64" + ))] + let direct = u8::from(darwin_tsd::active()); + #[cfg(not(all( + target_vendor = "apple", + target_arch = "aarch64", + target_pointer_width = "64" + )))] + let direct = 0u8; + eprintln!( + "[tls-hot] claimed={} published={} capacity={} direct_tsd={}", + claimed_slots(), + published_slots(), + HOT_SLOT_CAPACITY, + direct, + ); + } + // SAFETY: `report` is `extern "C"`, takes nothing and returns nothing. + unsafe { + libc::atexit(report); + } + }); +} + +/// The storage a [`crate::perry_thread_local`] declaration actually owns. +/// +/// `GUARD` is `needs_drop::() as usize`, filled in by the macro at each +/// declaration, and it is the whole reason this is a const-generic type rather +/// than a plain struct. +/// +/// * `T` owns something (`RefCell>`): `GUARD == 1`. The array's +/// element runs first — fields drop in declaration order, and `guard` is +/// declared first — so this thread's cached address is un-published *before* +/// `inner` is destroyed. A later access finds a null slot, falls back to the +/// real `thread_local!`, and gets std's "accessed during or after +/// destruction" panic instead of reading a dropped `HashMap`. The named-field +/// cache above has no such hook; this is strictly safer than it. +/// * `T` owns nothing (`Cell`): `GUARD == 0`. A zero-length array has no +/// drop glue, so `HotCell` has none either: no destructor is registered, the +/// `const`-init fast path in std is preserved, and a value that could always +/// be read during teardown still can be. Caching cannot make it dangle +/// because there is nothing to destroy. +pub struct HotCell { + guard: [SlotGuard; GUARD], + inner: T, +} + +impl HotCell { + pub const fn new(inner: T) -> Self { + Self { + guard: [const { SlotGuard::new() }; GUARD], + inner, + } + } + + /// The address the cache stores: the *value*, so [`HotKey`] never has to + /// know this type's layout — or its `GUARD`. + #[doc(hidden)] + pub fn value_addr(&self) -> *mut u8 { + &self.inner as *const T as *mut u8 + } + + /// Tell the teardown guard, if there is one, which slot to un-publish. + #[doc(hidden)] + pub fn arm_guard(&self, idx: u32) { + if let Some(guard) = self.guard.first() { + guard.idx.set(idx); + } + } +} + +/// Nulls this thread's cached pointer when the value it points into is about +/// to be destroyed. +struct SlotGuard { + idx: Cell, +} + +impl SlotGuard { + const fn new() -> Self { + Self { + idx: Cell::new(SLOT_UNASSIGNED), + } + } +} + +impl Drop for SlotGuard { + fn drop(&mut self) { + let idx = self.idx.get(); + if (idx as usize) < HOT_SLOT_CAPACITY { + hot().set_slot(idx, std::ptr::null_mut()); + } + } +} + +/// A thread-local whose address is cached in this thread's [`HotTls`]. +/// +/// Drop-in for `std::thread::LocalKey` at the call site: `with` and `try_with` +/// keep the same signatures, so converting a declaration converts every one of +/// its uses. +pub struct HotKey { + slot: &'static SlotId, + /// Resolves the owning `thread_local!` the ordinary way and returns the + /// address of its *value*. Cold path only — never called once the slot is + /// populated, so the indirect call never appears on a hot path. + resolve: fn() -> Result<*mut u8, std::thread::AccessError>, + /// Records the claimed index in this thread's teardown guard, if the value + /// has one. Generated alongside the storage, so it knows the `GUARD` that + /// `HotKey` deliberately does not. + arm_guard: fn(u32), + _not_send: std::marker::PhantomData<*const T>, +} + +// SAFETY: exactly `LocalKey`'s argument. `HotKey` is a handle, not storage: +// every path through it resolves the *calling* thread's own cell, so no `T` is +// ever observed from a thread other than the one that owns it. +unsafe impl Sync for HotKey {} + +impl HotKey { + #[doc(hidden)] + pub const fn new( + slot: &'static SlotId, + resolve: fn() -> Result<*mut u8, std::thread::AccessError>, + arm_guard: fn(u32), + ) -> Self { + Self { + slot, + resolve, + arm_guard, + _not_send: std::marker::PhantomData, + } + } + + /// Borrow this thread's value. + /// + /// The fast path is: one load of the declaration's index, the [`hot`] + /// cache (an `mrs` plus two loads on Apple aarch64, CSE'd across the whole + /// enclosing function), one load from the slot array. No call, so no + /// caller-saved register is clobbered at the site. + #[inline(always)] + pub fn with(&'static self, f: F) -> R + where + F: FnOnce(&T) -> R, + { + f(self.get()) + } + + /// As [`HotKey::with`], but reports rather than panics when this thread's + /// value is being or has been destroyed. + #[inline(always)] + pub fn try_with(&'static self, f: F) -> Result + where + F: FnOnce(&T) -> R, + { + let idx = self.slot.raw(); + if (idx as usize) < HOT_SLOT_CAPACITY { + let cell = hot().slot(idx); + if !cell.is_null() { + // SAFETY: see `value_of`. + return Ok(f(unsafe { Self::value_of(cell) })); + } + } + let cell = self.resolve_and_cache()?; + // SAFETY: see `value_of`. + Ok(f(unsafe { Self::value_of(cell) })) + } + + #[inline(always)] + fn get(&'static self) -> &'static T { + let idx = self.slot.raw(); + if (idx as usize) < HOT_SLOT_CAPACITY { + let cell = hot().slot(idx); + if !cell.is_null() { + // SAFETY: see `value_of`. + return unsafe { Self::value_of(cell) }; + } + } + self.get_slow() + } + + #[cold] + #[inline(never)] + fn get_slow(&'static self) -> &'static T { + let cell = self + .resolve_and_cache() + .expect("cannot access a Perry thread-local during or after thread destruction"); + // SAFETY: see `value_of`. + unsafe { Self::value_of(cell) } + } + + /// This declaration's claimed slot index, or a sentinel + /// `>= HOT_SLOT_CAPACITY` if it has none. Liveness instrumentation: a test + /// that does not check this passes identically when the cache is inert. + #[doc(hidden)] + pub fn slot_index(&'static self) -> u32 { + self.slot.raw() + } + + /// `value` is the address of this thread's `T`, published by this key. + /// + /// # Safety + /// `value` must have come from this key's slot or from its own `resolve`. + /// That is what makes the cross-cast the module docs warn about impossible + /// here: [`crate::perry_thread_local`] generates the storage, the resolver + /// and this key's `T` from one declaration, so there is no hand-written + /// pairing left to get wrong. + #[inline(always)] + unsafe fn value_of(value: *mut u8) -> &'static T { + // SAFETY: the caller guarantees provenance; the value outlives this + // thread's use of it (see the `SlotGuard` note on destruction). + unsafe { &*(value as *const T) } + } + + /// Resolve through the real `thread_local!`, claim this declaration's slot + /// if it has none yet, and publish the address for this thread. + #[cold] + #[inline(never)] + fn resolve_and_cache(&'static self) -> Result<*mut u8, std::thread::AccessError> { + // Resolve first, and outside the claim lock: initialising the value can + // run arbitrary runtime code, including other `perry_thread_local!` + // first touches. + let value = (self.resolve)()?; + let mut idx = self.slot.raw(); + if idx == SLOT_UNASSIGNED { + idx = self.slot.claim(); + } + if (idx as usize) < HOT_SLOT_CAPACITY { + // Arm before publishing: after this store any thread-teardown of + // the value un-publishes the slot it is about to invalidate. + (self.arm_guard)(idx); + hot().set_slot(idx, value); + } + Ok(value) + } +} + +/// Declare a thread-local that is on the fast path **by default**. +/// +/// Same syntax as `std::thread_local!`, same `with` / `try_with` at every call +/// site — the only difference is that the address of the value lands in this +/// thread's [`HotTls`] cache, so reads cost loads instead of a `_tlv_get_addr` +/// call on Darwin. +/// +/// # Why this is the default rather than an allowlist +/// +/// The named fields at the top of this file are an opt-in list of sixteen, +/// curated against whichever workload was profiled last. Every hot path the +/// list did not anticipate silently paid full price: `churn` read 0% of +/// `_tlv_get_addr` for months while `asyncpipe` — Map/Set registries, buffer +/// brands, descriptor state, none of them on the list — paid 20.5%. Adding a +/// field took four manual steps including a hand-written test, which does not +/// scale to ~520 declarations and, worse, gets the *default* wrong: forgetting +/// the steps produces a working slow path rather than a build error. +/// +/// Here there is nothing to wire. The declaration generates its own storage, +/// its own resolver and its own [`HotKey`], so a mis-pairing of slot and type — +/// the hazard the untyped named slots need `cached_addresses_match_thread_locals` +/// to catch — cannot be expressed. +/// +/// # Forms +/// +/// ```ignore +/// crate::perry_thread_local! { +/// static COUNTER: Cell = const { Cell::new(0) }; +/// static REGISTRY: RefCell> = RefCell::new(HashMap::new()); +/// } +/// ``` +/// +/// Both forms behave exactly as `std::thread_local!`'s do, including whether a +/// destructor is registered: the teardown guard is present iff +/// `needs_drop::()`, so a `Cell` keeps std's drop-free `const` path and +/// a `RefCell>` gets un-published before it is destroyed. +#[macro_export] +macro_rules! perry_thread_local { + () => {}; + + // `= const { ... }` + ($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty = const $init:block; $($rest:tt)*) => { + $crate::__perry_thread_local_one! { $(#[$attr])* $vis $name, $t, const $init } + $crate::perry_thread_local!($($rest)*); + }; + + // `= expr` + ($(#[$attr:meta])* $vis:vis static $name:ident: $t:ty = $init:expr; $($rest:tt)*) => { + $crate::__perry_thread_local_one! { $(#[$attr])* $vis $name, $t, expr ($init) } + $crate::perry_thread_local!($($rest)*); + }; +} + +/// One declaration. Split out only so the two init forms share everything but +/// the line that hands the initialiser to `std::thread_local!`. +#[doc(hidden)] +#[macro_export] +macro_rules! __perry_thread_local_one { + ($(#[$attr:meta])* $vis:vis $name:ident, $t:ty, $($init:tt)+) => { + $(#[$attr])* + $vis static $name: $crate::tls_hot::HotKey<$t> = { + static SLOT: $crate::tls_hot::SlotId = $crate::tls_hot::SlotId::new(); + // `GUARD` is 1 exactly when `$t` has drop glue, so the guard — + // and with it the thread-local's destructor — exists exactly when + // a cached address could otherwise outlive the value. + type Storage = $crate::tls_hot::HotCell<$t, { ::core::mem::needs_drop::<$t>() as usize }>; + $crate::__perry_thread_local_storage!(Storage, $($init)+); + fn resolve() -> ::core::result::Result<*mut u8, ::std::thread::AccessError> { + STORAGE.try_with(|cell| cell.value_addr()) + } + fn arm_guard(idx: u32) { + let _ = STORAGE.try_with(|cell| cell.arm_guard(idx)); + } + $crate::tls_hot::HotKey::new(&SLOT, resolve, arm_guard) + }; + }; +} + +#[doc(hidden)] +#[macro_export] +macro_rules! __perry_thread_local_storage { + ($storage:ty, const $init:block) => { + ::std::thread_local! { + static STORAGE: $storage = const { <$storage>::new($init) }; + } + }; + ($storage:ty, expr ($init:expr)) => { + ::std::thread_local! { + static STORAGE: $storage = <$storage>::new($init); + } + }; +} + #[cfg(test)] mod tests { /// Every cached address must equal the address of the `thread_local!` it @@ -602,4 +1106,241 @@ mod tests { "two threads resolved the same temp-root address" ); } + + // ----------------------------------------------------------------- + // Generic slots + // ----------------------------------------------------------------- + + crate::perry_thread_local! { + static PROBE_CONST: std::cell::Cell = const { std::cell::Cell::new(0) }; + static PROBE_EXPR: std::cell::RefCell> = std::cell::RefCell::new(Vec::new()); + static PROBE_SECOND: std::cell::Cell = const { std::cell::Cell::new(0) }; + } + + /// The address a `perry_thread_local!` hands out is this thread's real + /// storage, and it is stable across accesses. + /// + /// The named slots need `cached_addresses_match_thread_locals` because a + /// human writes the pairing; here the storage, the resolver and the key's + /// `T` all come from one declaration, so this is a liveness check (the slot + /// really is being used) rather than a correctness one. + #[test] + fn a_generic_slot_is_this_threads_storage() { + PROBE_CONST.with(|c| c.set(0x5eed)); + let first = PROBE_CONST.with(|c| c as *const _ as usize); + let second = PROBE_CONST.with(|c| c as *const _ as usize); + assert_eq!(first, second, "the cached address moved between accesses"); + assert_eq!(PROBE_CONST.with(|c| c.get()), 0x5eed); + + // Liveness: the access above must actually have gone through a slot. + // Every other assertion in this module passes identically whether the + // cache is used or every read falls back to `_tlv_get_addr`, so without + // this one the whole mechanism could be inert and nothing would go red. + let idx = PROBE_CONST.slot_index(); + assert!( + (idx as usize) < super::HOT_SLOT_CAPACITY, + "the declaration never claimed a slot (idx {idx})", + ); + assert_eq!( + super::hot().slot(idx) as usize, + first, + "the slot does not hold the address `with` handed out", + ); + } + + /// Distinct declarations must never share a slot: that is the one way a + /// generic slot could hand out a correctly-typed reference to the wrong + /// object, which is the hazard the module docs are about. + #[test] + fn distinct_declarations_do_not_share_a_slot() { + PROBE_CONST.with(|c| c.set(11)); + PROBE_SECOND.with(|c| c.set(22)); + assert_eq!(PROBE_CONST.with(|c| c.get()), 11); + assert_eq!(PROBE_SECOND.with(|c| c.get()), 22); + let a = PROBE_CONST.with(|c| c as *const _ as usize); + let b = PROBE_SECOND.with(|c| c as *const _ as usize); + assert_ne!(a, b, "two declarations resolved to one address"); + } + + /// Each thread resolves its own storage, and a worker's slot must not + /// leak into the parent's cache. + #[test] + fn a_generic_slot_is_per_thread() { + PROBE_EXPR.with(|v| v.borrow_mut().push(1)); + let mine = PROBE_EXPR.with(|v| v as *const _ as usize); + let theirs = std::thread::spawn(|| { + PROBE_EXPR.with(|v| { + assert!( + v.borrow().is_empty(), + "a worker inherited the parent thread's value" + ); + v.borrow_mut().push(2); + }); + PROBE_EXPR.with(|v| v as *const _ as usize) + }) + .join() + .expect("probe thread panicked"); + assert_ne!(mine, theirs, "two threads shared one slot's storage"); + assert_eq!(PROBE_EXPR.with(|v| v.borrow().clone()), vec![1]); + } + + /// Hammer the claim path from many threads at once: a lost race that + /// leaked an index would show up as capacity draining far past the number + /// of declarations. + #[test] + fn concurrent_first_touch_claims_one_slot_per_declaration() { + let before = super::claimed_slots(); + let workers: Vec<_> = (0..8) + .map(|_| { + std::thread::spawn(|| { + PROBE_CONST.with(|c| c.get()); + PROBE_EXPR.with(|v| v.borrow().len()); + PROBE_SECOND.with(|c| c.get()); + }) + }) + .collect(); + for w in workers { + w.join().expect("probe thread panicked"); + } + let after = super::claimed_slots(); + assert!( + after - before <= 3, + "8 threads first-touching 3 declarations claimed {} slots", + after - before + ); + assert!( + (after as usize) < super::HOT_SLOT_CAPACITY, + "slot capacity {} exhausted at {after} claims", + super::HOT_SLOT_CAPACITY + ); + } + + /// A value with a destructor must stop being served from the cache the + /// moment its thread starts destroying it. + /// + /// The ordering is the point. std runs thread-local destructors in reverse + /// registration order, so touching `AFTER_PROBE` *first* and `PROBE_EXPR` + /// second puts `PROBE_EXPR`'s destructor ahead of `AFTER_PROBE`'s: by the + /// time `AFTER_PROBE` drops and re-reads the key, the guard must already + /// have un-published it. Without the guard this test reads a dropped `Vec` + /// — the exact use-after-free the named-field cache has no defence against. + #[test] + fn teardown_unpublishes_a_dropping_value() { + use std::sync::atomic::{AtomicU8, Ordering}; + static OBSERVED: AtomicU8 = AtomicU8::new(0); + const UNSEEN: u8 = 0; + const REPORTED_DESTROYED: u8 = 1; + const SERVED_STALE: u8 = 2; + + struct AfterProbe; + impl Drop for AfterProbe { + fn drop(&mut self) { + let state = match PROBE_EXPR.try_with(|v| v.borrow().len()) { + Ok(_) => SERVED_STALE, + Err(_) => REPORTED_DESTROYED, + }; + OBSERVED.store(state, Ordering::SeqCst); + } + } + thread_local! { + static AFTER_PROBE: AfterProbe = const { AfterProbe }; + } + + std::thread::spawn(|| { + // Registration order: AFTER_PROBE, then PROBE_EXPR's storage. + AFTER_PROBE.with(|_| {}); + PROBE_EXPR.with(|v| v.borrow_mut().push(7)); + }) + .join() + .expect("probe thread panicked"); + + assert_eq!( + OBSERVED.load(Ordering::SeqCst), + REPORTED_DESTROYED, + "a destroyed thread-local was still served from the hot cache \ + (0 = the probe never ran, 2 = it read the dropped value)", + ); + assert_ne!(OBSERVED.load(Ordering::SeqCst), UNSEEN); + } + + /// Real converted declarations, across many short-lived threads. + /// + /// `perry/thread`'s `spawn` and `parallelMap` run JS on OS threads with + /// their own arenas, so every converted declaration is resolved, published + /// and destroyed once per worker. This drives that cycle 64 times over the + /// registry probes the profiles named — the ones `asyncpipe` and `interp` + /// spend their `_tlv_get_addr` on — and asserts both that nothing faults + /// and that repeated thread turnover cannot drain slots: an index is + /// claimed per *declaration*, not per thread. + #[test] + fn converted_declarations_survive_thread_turnover() { + fn touch_the_converted_registries() -> usize { + let mut seen = 0; + for probe in [0usize, 1, 0x1000, usize::MAX / 2] { + seen += usize::from(crate::map::is_registered_map(probe)); + seen += usize::from(crate::set::is_registered_set(probe)); + seen += usize::from(crate::buffer::is_registered_buffer(probe)); + seen += usize::from(crate::symbol::is_registered_symbol(probe)); + seen += usize::from(crate::regex::is_regex_pointer(probe as *const u8)); + } + seen + } + + touch_the_converted_registries(); + let after_main = super::claimed_slots(); + + for _ in 0..8 { + let batch: Vec<_> = (0..8) + .map(|_| std::thread::spawn(touch_the_converted_registries)) + .collect(); + for worker in batch { + worker.join().expect("worker thread panicked"); + } + } + + let after_workers = super::claimed_slots(); + // A TOLERANCE, not equality, and the reason is the measurement rather + // than the mechanism: `claimed_slots()` is PROCESS-global, so any other + // test in this binary that touches a converted declaration for the + // first time lands a claim inside this window. Under `--test-threads=1` + // the delta is exactly 0; in parallel it is 0 or a stray 1-2 from a + // neighbour. Asserting equality made this fail 6/6 under load while + // passing 3/3 in isolation and 1/1 single-threaded. + // + // The tolerance still separates the two outcomes by two orders of + // magnitude: per-THREAD claiming — the bug this test exists to catch — + // would add 5 declarations x 64 workers = 320, not 1. + const TURNOVER_CLAIM_TOLERANCE: u32 = 8; + let extra = after_workers.saturating_sub(after_main); + assert!( + extra <= TURNOVER_CLAIM_TOLERANCE, + "64 worker threads claimed {extra} extra slots (tolerance \ + {TURNOVER_CLAIM_TOLERANCE}); indices must be per declaration, not \ + per thread — per-thread claiming would add 5 x 64 = 320", + ); + assert!( + (after_workers as usize) < super::HOT_SLOT_CAPACITY, + "capacity {} exhausted at {after_workers}", + super::HOT_SLOT_CAPACITY, + ); + // And the parent thread's own cache is still serving after all that. + assert_eq!(touch_the_converted_registries(), 0); + } + + /// The guard exists exactly when the value has something to destroy — that + /// is what keeps drop-free declarations off std's destructor path. + #[test] + fn the_guard_tracks_drop_glue() { + assert_eq!( + std::mem::size_of::, 0>>(), + std::mem::size_of::>(), + "a drop-free declaration paid for a guard", + ); + assert!(!std::mem::needs_drop::< + super::HotCell, 0>, + >()); + assert!(std::mem::needs_drop::< + super::HotCell>, 1>, + >()); + } } diff --git a/crates/perry-runtime/src/typedarray/mod.rs b/crates/perry-runtime/src/typedarray/mod.rs index 8c6ec35bda..6efac89f15 100644 --- a/crates/perry-runtime/src/typedarray/mod.rs +++ b/crates/perry-runtime/src/typedarray/mod.rs @@ -168,7 +168,7 @@ pub struct TypedArrayHeader { pub _pad: [u8; 6], } -thread_local! { +crate::perry_thread_local! { /// Address -> kind, so we can detect typed arrays at format/instanceof time. /// PtrHasher (Fibonacci-multiplicative + xorshift): heap pointers don't /// need SipHash. Hot on `is_registered_buffer`-adjacent dispatch paths diff --git a/crates/perry-runtime/src/typedarray_props.rs b/crates/perry-runtime/src/typedarray_props.rs index 2e409f141f..07f13cd635 100644 --- a/crates/perry-runtime/src/typedarray_props.rs +++ b/crates/perry-runtime/src/typedarray_props.rs @@ -6,7 +6,7 @@ use crate::typedarray::{ js_typed_array_get, js_typed_array_set, lookup_typed_array_kind, TypedArrayHeader, }; -thread_local! { +crate::perry_thread_local! { static TYPED_ARRAY_OWN_PROPS: RefCell>> = RefCell::new(crate::fast_hash::new_ptr_hash_map()); } @@ -313,7 +313,7 @@ fn throw_typed_array_define_error(message: String) -> ! { throw_type_error(message.as_bytes()) } -thread_local! { +crate::perry_thread_local! { /// Typed arrays marked non-extensible by `Object.preventExtensions`. /// A SIDE TABLE, not the GC-header flag: small typed arrays are plain /// `alloc`ed without a `GcHeader`, so flag reads/writes at `addr - 8` diff --git a/crates/perry-runtime/src/value/to_string.rs b/crates/perry-runtime/src/value/to_string.rs index 85f6f2fe4d..7705ef6ff9 100644 --- a/crates/perry-runtime/src/value/to_string.rs +++ b/crates/perry-runtime/src/value/to_string.rs @@ -4,7 +4,7 @@ use super::*; use std::cell::Cell; use std::sync::atomic::Ordering; -thread_local! { +crate::perry_thread_local! { /// Re-entrancy guard for `OrdinaryToPrimitive(string)`. A user /// `toString`/`valueOf` whose body coerces `this` back to a string /// (e.g. `toString() { return "" + this; }`) would recurse forever; diff --git a/scripts/check_thread_locals.py b/scripts/check_thread_locals.py new file mode 100755 index 0000000000..6b801c6656 --- /dev/null +++ b/scripts/check_thread_locals.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +"""Keep `thread_local!` from silently reintroducing `_tlv_get_addr` cost (#7469). + +WHY THIS EXISTS +=============== + +On Darwin every `thread_local!` access is an out-of-line call to +`_tlv_get_addr` in libdyld, and the runtime resolves thread-locals often enough +that the call was the single largest symbol in the worst-performing program in +the corpus (20.5% of `asyncpipe`). `crates/perry-runtime/src/tls_hot.rs` fixes +the *mechanism*; this script fixes the *policy*. + +The mechanism has been fixed three times and regressed three times, because +coverage was an opt-in allowlist of sixteen hand-wired fields curated against +whichever workload was profiled last. `churn` — the profiled one — read 0% +forever. Every path the list did not anticipate paid full price, silently, +because forgetting to add a field produces a *working slow path* rather than a +build error. + +`crate::perry_thread_local!` inverts that: a declaration written with it is on +the fast path with no wiring, no hand-written test and no list. This script is +what makes that the default rather than a suggestion — a new raw `thread_local!` +is a build error unless it is deliberately recorded here as cold. + +THIS CHECK IS DESIGNED TO BE ABLE TO FAIL +========================================= + +* A file gaining a raw `thread_local!` fails (the regression this exists for). +* A file *losing* one also fails: a stale allowlist entry is an entry nobody + has to justify any more, and the `gc_root_dominance_allowlist.json` rule + applies — an entry that matches nothing must be deleted, not left to rot. +* `--self-test` runs both directions against synthetic trees, so the checker + itself cannot quietly stop being able to say no. + +WHAT IT DOES NOT DO +=================== + +It cannot tell a hot cold-listed declaration from a genuinely cold one. That is +what the runtime budget gate (`scripts/tls_budget_gate.sh`, measuring +`_tlv_get_addr`'s share of two programs whose paths are deliberately *not* in +the sixteen named slots) is for. This one is the fast, hermetic half. + +USAGE +===== + + scripts/check_thread_locals.py # verify (CI) + scripts/check_thread_locals.py --update # rewrite the allowlist + scripts/check_thread_locals.py --self-test +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import tempfile +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +ALLOWLIST = REPO / "scripts" / "thread_local_cold_allowlist.json" +CRATES = ["crates/perry-runtime/src"] + +# A raw declaration: `thread_local!` or `std::thread_local!` at the start of a +# statement. `crate::perry_thread_local! {` does not match, and neither does the +# one inside the macro's own expansion in tls_hot.rs (excluded by path below). +RAW_RE = re.compile(r"(?m)^[ \t]*(?:std::)?thread_local!\s*\{") +HOT_RE = re.compile(r"(?m)^[ \t]*(?:crate::)?perry_thread_local!\s*\{") +# One `static NAME: Ty = …;` inside a block. +DECL_RE = re.compile( + r"(?m)^\s*(?:#\[[^\]]*\]\s*)*(?:pub(?:\([^)]*\))?\s+)?static\s+[A-Za-z_0-9]+\s*:" +) + +# `tls_hot.rs` defines the mechanism: its own `thread_local!`s are the cache +# itself and the macro's expansion target, so they cannot be written with the +# macro without infinite regress. +EXCLUDED = {"crates/perry-runtime/src/tls_hot.rs"} + + +def block_bodies(src: str, pattern: re.Pattern[str]) -> list[str]: + """Bodies of every macro block `pattern` starts, brace-matched.""" + bodies = [] + for m in pattern.finditer(src): + i = src.index("{", m.start()) + depth = 0 + j = i + while j < len(src): + if src[j] == "{": + depth += 1 + elif src[j] == "}": + depth -= 1 + if depth == 0: + break + j += 1 + bodies.append(src[i + 1 : j]) + return bodies + + +def scan(root: Path, crates: list[str]) -> tuple[dict[str, int], int]: + """Raw `thread_local!` blocks per file, and total hot declarations.""" + raw: dict[str, int] = {} + hot_declarations = 0 + for crate in crates: + base = root / crate + for dirpath, _dirs, files in os.walk(base): + for name in sorted(files): + if not name.endswith(".rs"): + continue + path = Path(dirpath) / name + rel = str(path.relative_to(root)) + src = path.read_text() + hot_declarations += sum( + len(DECL_RE.findall(body)) for body in block_bodies(src, HOT_RE) + ) + if rel in EXCLUDED: + continue + count = len(RAW_RE.findall(src)) + if count: + raw[rel] = count + return raw, hot_declarations + + +def hot_slot_capacity(root: Path) -> int: + src = (root / "crates/perry-runtime/src/tls_hot.rs").read_text() + m = re.search(r"pub const HOT_SLOT_CAPACITY: usize = (\d+);", src) + if not m: + raise SystemExit("could not read HOT_SLOT_CAPACITY from tls_hot.rs") + return int(m.group(1)) + + +def verify(root: Path, crates: list[str], allowlist_path: Path) -> list[str]: + raw, hot_declarations = scan(root, crates) + recorded = json.loads(allowlist_path.read_text())["files"] + problems = [] + + for rel, count in sorted(raw.items()): + if rel not in recorded: + problems.append( + f"{rel}: {count} raw `thread_local!` block(s), none allowed.\n" + f" Use `crate::perry_thread_local!` — same syntax, same " + f"`.with()` at every call site, and the address lands in this " + f"thread's hot cache instead of costing a `_tlv_get_addr` call " + f"(#7469). If the declaration really is cold, record it:\n" + f" scripts/check_thread_locals.py --update" + ) + elif recorded[rel] != count: + direction = "gained" if count > recorded[rel] else "lost" + problems.append( + f"{rel}: {direction} raw `thread_local!` blocks " + f"({recorded[rel]} recorded, {count} found). " + f"Convert it, or run --update to re-record." + ) + + for rel in sorted(recorded): + if rel not in raw: + problems.append( + f"{rel}: recorded as having {recorded[rel]} cold " + f"`thread_local!` block(s), but has none. A stale entry is one " + f"nobody has to justify — delete it with --update." + ) + + capacity = hot_slot_capacity(root) + if hot_declarations >= capacity: + problems.append( + f"{hot_declarations} `perry_thread_local!` declarations vs " + f"HOT_SLOT_CAPACITY {capacity}. Slot exhaustion is *correct* but " + f"silent — the declarations past the ceiling fall back to " + f"`_tlv_get_addr` forever. Raise HOT_SLOT_CAPACITY." + ) + return problems + + +def write_allowlist(root: Path, crates: list[str], allowlist_path: Path) -> None: + raw, hot_declarations = scan(root, crates) + allowlist_path.write_text( + json.dumps( + { + "_comment": ( + "Files still declaring raw `thread_local!`. Every entry is a " + "declaration that pays `_tlv_get_addr` on Darwin; the count is " + "a ratchet, so adding one to an already-listed file fails too. " + "New code should use `crate::perry_thread_local!` — see " + "crates/perry-runtime/src/tls_hot.rs. Regenerate with " + "scripts/check_thread_locals.py --update." + ), + "_hot_declarations": hot_declarations, + "files": dict(sorted(raw.items())), + }, + indent=2, + ) + + "\n" + ) + + +def self_test() -> int: + """Prove the checker can say no, in both directions.""" + failures = [] + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + src_dir = root / "crates/perry-runtime/src" + src_dir.mkdir(parents=True) + (src_dir / "tls_hot.rs").write_text( + "pub const HOT_SLOT_CAPACITY: usize = 768;\n" + "thread_local! { static HOT: u8 = const { 0 }; }\n" + ) + (src_dir / "cold.rs").write_text("thread_local! { static A: u8 = const { 0 }; }\n") + (src_dir / "hot.rs").write_text( + "crate::perry_thread_local! { static B: u8 = const { 0 }; }\n" + ) + allowlist = root / "allow.json" + + write_allowlist(root, CRATES, allowlist) + recorded = json.loads(allowlist.read_text()) + if "crates/perry-runtime/src/tls_hot.rs" in recorded["files"]: + failures.append("tls_hot.rs must be excluded from the allowlist") + if recorded["_hot_declarations"] != 1: + failures.append( + f"expected 1 hot declaration, counted {recorded['_hot_declarations']}" + ) + if verify(root, CRATES, allowlist): + failures.append("a freshly written allowlist must verify clean") + + # 1. A new raw declaration in an unlisted file must fail. + (src_dir / "new.rs").write_text("thread_local! { static C: u8 = const { 0 }; }\n") + if not verify(root, CRATES, allowlist): + failures.append("a new raw `thread_local!` in an unlisted file passed") + (src_dir / "new.rs").unlink() + + # 2. A second raw declaration in an already-listed file must fail. + (src_dir / "cold.rs").write_text( + "thread_local! { static A: u8 = const { 0 }; }\n" + "thread_local! { static D: u8 = const { 0 }; }\n" + ) + if not verify(root, CRATES, allowlist): + failures.append("a raw `thread_local!` added to a listed file passed") + + # 3. A stale entry must fail. + (src_dir / "cold.rs").write_text( + "crate::perry_thread_local! { static A: u8 = const { 0 }; }\n" + ) + if not verify(root, CRATES, allowlist): + failures.append("a stale allowlist entry passed") + + # 4. Blowing the slot ceiling must fail. + write_allowlist(root, CRATES, allowlist) + (src_dir / "hot.rs").write_text( + "crate::perry_thread_local! {\n" + + "".join(f" static H{i}: u8 = const {{ 0 }};\n" for i in range(800)) + + "}\n" + ) + if not any("HOT_SLOT_CAPACITY" in p for p in verify(root, CRATES, allowlist)): + failures.append("exceeding HOT_SLOT_CAPACITY passed") + + for f in failures: + print(f"SELF-TEST FAILED: {f}", file=sys.stderr) + if failures: + return 1 + print("self-test: the checker can fail in all four directions") + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--update", action="store_true", help="rewrite the allowlist") + ap.add_argument("--self-test", action="store_true", help="prove the checker can fail") + args = ap.parse_args() + + if args.self_test: + return self_test() + if args.update: + write_allowlist(REPO, CRATES, ALLOWLIST) + print(f"wrote {ALLOWLIST.relative_to(REPO)}") + return 0 + + problems = verify(REPO, CRATES, ALLOWLIST) + if problems: + print("thread-local policy check FAILED:\n", file=sys.stderr) + for p in problems: + print(f" {p}\n", file=sys.stderr) + return 1 + raw, hot_declarations = scan(REPO, CRATES) + print( + f"thread-local policy OK: {hot_declarations} hot declarations, " + f"{sum(raw.values())} raw blocks in {len(raw)} recorded cold files, " + f"capacity {hot_slot_capacity(REPO)}" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/thread_local_cold_allowlist.json b/scripts/thread_local_cold_allowlist.json new file mode 100644 index 0000000000..afde09619b --- /dev/null +++ b/scripts/thread_local_cold_allowlist.json @@ -0,0 +1,107 @@ +{ + "_comment": "Files still declaring raw `thread_local!`. Every entry is a declaration that pays `_tlv_get_addr` on Darwin; the count is a ratchet, so adding one to an already-listed file fails too. New code should use `crate::perry_thread_local!` \u2014 see crates/perry-runtime/src/tls_hot.rs. Regenerate with scripts/check_thread_locals.py --update.", + "_hot_declarations": 157, + "files": { + "crates/perry-runtime/src/agent.rs": 1, + "crates/perry-runtime/src/arena/block.rs": 3, + "crates/perry-runtime/src/arena/page_meta.rs": 2, + "crates/perry-runtime/src/arena/quarantine.rs": 1, + "crates/perry-runtime/src/async_context.rs": 2, + "crates/perry-runtime/src/async_hooks.rs": 3, + "crates/perry-runtime/src/builtins/arithmetic.rs": 1, + "crates/perry-runtime/src/builtins/console.rs": 2, + "crates/perry-runtime/src/builtins/formatting.rs": 5, + "crates/perry-runtime/src/builtins/formatting/boxed_primitives.rs": 1, + "crates/perry-runtime/src/builtins/globals.rs": 2, + "crates/perry-runtime/src/bun_ffi/types.rs": 1, + "crates/perry-runtime/src/child_process/reactor.rs": 1, + "crates/perry-runtime/src/child_process/v8_serde.rs": 1, + "crates/perry-runtime/src/closure/dispatch/errors.rs": 1, + "crates/perry-runtime/src/cluster.rs": 2, + "crates/perry-runtime/src/dyn_eval/bridge.rs": 1, + "crates/perry-runtime/src/dyn_eval/env.rs": 1, + "crates/perry-runtime/src/dyn_eval/interp.rs": 1, + "crates/perry-runtime/src/dyn_eval/mod.rs": 1, + "crates/perry-runtime/src/eh.rs": 1, + "crates/perry-runtime/src/eh_walker.rs": 1, + "crates/perry-runtime/src/error.rs": 2, + "crates/perry-runtime/src/event_pump.rs": 1, + "crates/perry-runtime/src/fs/callbacks.rs": 1, + "crates/perry-runtime/src/fs/dir_glob_watch/watch.rs": 1, + "crates/perry-runtime/src/fs/filehandle.rs": 1, + "crates/perry-runtime/src/fs/mod.rs": 1, + "crates/perry-runtime/src/fs/stream.rs": 1, + "crates/perry-runtime/src/gc/barrier.rs": 2, + "crates/perry-runtime/src/gc/barrier_arming.rs": 1, + "crates/perry-runtime/src/gc/cycle.rs": 2, + "crates/perry-runtime/src/gc/dirty_page_cache.rs": 1, + "crates/perry-runtime/src/gc/fromspace_scan.rs": 1, + "crates/perry-runtime/src/gc/layout.rs": 2, + "crates/perry-runtime/src/gc/layout_tables.rs": 1, + "crates/perry-runtime/src/gc/malloc.rs": 2, + "crates/perry-runtime/src/gc/mod.rs": 3, + "crates/perry-runtime/src/gc/old_free.rs": 1, + "crates/perry-runtime/src/gc/oldgen_defrag.rs": 1, + "crates/perry-runtime/src/gc/policy.rs": 10, + "crates/perry-runtime/src/gc/promote_in_place.rs": 2, + "crates/perry-runtime/src/gc/roots/scan_mode.rs": 1, + "crates/perry-runtime/src/gc/roots/shadow_stack.rs": 2, + "crates/perry-runtime/src/gc/roots/temp_roots.rs": 1, + "crates/perry-runtime/src/gc/scan_fallback.rs": 1, + "crates/perry-runtime/src/gc/shape_install.rs": 2, + "crates/perry-runtime/src/gc/telemetry.rs": 3, + "crates/perry-runtime/src/gc/tenuring.rs": 1, + "crates/perry-runtime/src/gc/tests/copying.rs": 1, + "crates/perry-runtime/src/gc/tests/roots.rs": 1, + "crates/perry-runtime/src/gc/tests/runtime_roots.rs": 4, + "crates/perry-runtime/src/gc/tests/runtime_roots/transient_handles.rs": 1, + "crates/perry-runtime/src/gc/tests/support.rs": 1, + "crates/perry-runtime/src/gc/trace.rs": 2, + "crates/perry-runtime/src/gc/zeal.rs": 3, + "crates/perry-runtime/src/intl/number_format.rs": 1, + "crates/perry-runtime/src/iter_result.rs": 1, + "crates/perry-runtime/src/json/mod.rs": 1, + "crates/perry-runtime/src/json/raw_json.rs": 1, + "crates/perry-runtime/src/json_tape.rs": 3, + "crates/perry-runtime/src/json_tape_store.rs": 1, + "crates/perry-runtime/src/media_playback.rs": 1, + "crates/perry-runtime/src/native_arena.rs": 1, + "crates/perry-runtime/src/node_http2_constants.rs": 1, + "crates/perry-runtime/src/node_inspector.rs": 1, + "crates/perry-runtime/src/node_repl.rs": 1, + "crates/perry-runtime/src/node_stream_constructors.rs": 2, + "crates/perry-runtime/src/node_stream_tests.rs": 1, + "crates/perry-runtime/src/node_submodules/blob.rs": 1, + "crates/perry-runtime/src/node_submodules/diagnostics.rs": 6, + "crates/perry-runtime/src/node_submodules/diagnostics_tail.rs": 1, + "crates/perry-runtime/src/node_submodules/mod.rs": 1, + "crates/perry-runtime/src/node_submodules/test.rs": 1, + "crates/perry-runtime/src/node_submodules/test_once_unit_tests.rs": 1, + "crates/perry-runtime/src/node_submodules/test_property.rs": 1, + "crates/perry-runtime/src/node_submodules/tests.rs": 1, + "crates/perry-runtime/src/node_submodules/trace_events.rs": 1, + "crates/perry-runtime/src/object/native_module/callable_exports.rs": 2, + "crates/perry-runtime/src/object/spill.rs": 2, + "crates/perry-runtime/src/os/os_process_emitter.rs": 1, + "crates/perry-runtime/src/os_process_streams.rs": 1, + "crates/perry-runtime/src/per_test_global.rs": 1, + "crates/perry-runtime/src/perf_hooks.rs": 3, + "crates/perry-runtime/src/process.rs": 2, + "crates/perry-runtime/src/process/env_misc.rs": 3, + "crates/perry-runtime/src/process/permission.rs": 1, + "crates/perry-runtime/src/process/report.rs": 1, + "crates/perry-runtime/src/proxy.rs": 1, + "crates/perry-runtime/src/pty/reactor.rs": 1, + "crates/perry-runtime/src/readline_helpers.rs": 1, + "crates/perry-runtime/src/static_plugins.rs": 1, + "crates/perry-runtime/src/timer.rs": 1, + "crates/perry-runtime/src/tty.rs": 1, + "crates/perry-runtime/src/typedarray_view.rs": 1, + "crates/perry-runtime/src/util_debuglog.rs": 1, + "crates/perry-runtime/src/util_promisify.rs": 1, + "crates/perry-runtime/src/v8.rs": 2, + "crates/perry-runtime/src/wasi.rs": 1, + "crates/perry-runtime/src/weakref.rs": 1, + "crates/perry-runtime/src/web_storage.rs": 1 + } +} diff --git a/scripts/tls_budget_check.py b/scripts/tls_budget_check.py new file mode 100755 index 0000000000..90e7269003 --- /dev/null +++ b/scripts/tls_budget_check.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +"""Verdict logic for the `_tlv_get_addr` budget gate (#7469). + +WHY THIS EXISTS +=============== + +On Darwin every `thread_local!` access is an out-of-line call to +`_tlv_get_addr`. `crates/perry-runtime/src/tls_hot.rs` collapses those calls to +loads, and it has done so three times: measured 0% of `churn_alloc` after +#7565, 8-9% a while later, 11% on `interp`/`retain`, and 20.5% of `asyncpipe` +by v0.5.1434. Nothing watched it. This is what watches it. + +THE VACUITY THAT MATTERS HERE +============================= + +The obvious gate — profile `churn_alloc`, assert `_tlv_get_addr` is small — +passes forever while the real cost grows, because `churn`'s thread-locals are +exactly the sixteen the named-field cache was curated for. It would be a gate +that cannot fail, of CLAUDE.md's fourth kind: green because its subject never +ran. + +So this checker refuses to return a verdict unless the run proves three things: + +1. The profile is real: enough samples, and enough *distinct* perry-runtime + symbols that it is a broad runtime workload rather than a tight loop with + one inlined allocation site. +2. The hot-cache mechanism was live: `PERRY_TLS_HOT_STATS=1` reports + `direct_tsd=1` (otherwise `hot()` is itself paying `_tlv_get_addr`, the + whole mechanism is inert, and a low share would mean the program resolved + nothing) and `claimed` above a floor. +3. The program exercised paths *outside* the sixteen named slots. `claimed` + counts generic-slot declarations that were actually resolved; `churn` drives + it to a handful, `asyncpipe` and `interp` to dozens. A subject that cannot + clear the floor is the wrong subject, and says so. + +`--self-test` drives every one of those rejections plus the budget itself, so +the checker cannot quietly stop being able to say no. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from collections import Counter + +LINE_RE = re.compile(r"^([ +!:|]*)(\d+) (.*?) \(in ([^)]*)\)") +STATS_RE = re.compile( + r"\[tls-hot\] claimed=(\d+) published=(\d+) capacity=(\d+) direct_tsd=(\d+)" +) +TARGET = "_tlv_get_addr" + + +class Profile: + def __init__(self, root: int, target: int, symbols: Counter, callers: Counter): + self.root = root + self.target = target + self.symbols = symbols + self.callers = callers + + @property + def share(self) -> float: + return 100.0 * self.target / self.root if self.root else 0.0 + + +def parse_report(text: str) -> Profile: + stack: list[tuple[int, str]] = [] + callers: Counter = Counter() + symbols: Counter = Counter() + root = 0 + target = 0 + started = False + for raw in text.splitlines(): + if raw.startswith("Call graph:"): + started = True + continue + if not started: + continue + if raw.startswith("Binary Images:") or raw.startswith("Total number"): + break + m = LINE_RE.match(raw) + if not m: + continue + indent, count, sym = len(m.group(1)), int(m.group(2)), m.group(3) + while stack and stack[-1][0] >= indent: + stack.pop() + if not stack and sym == "start": + root += count + if "perry_runtime" in sym or sym.startswith("js_"): + symbols[sym] += count + if sym.startswith(TARGET): + target += count + callers[stack[-1][1] if stack else ""] += count + stack.append((indent, sym)) + return Profile(root, target, symbols, callers) + + +def parse_stats(text: str) -> dict[str, int] | None: + m = STATS_RE.search(text) + if not m: + return None + return { + "claimed": int(m.group(1)), + "published": int(m.group(2)), + "capacity": int(m.group(3)), + "direct_tsd": int(m.group(4)), + } + + +def check( + report: str, + stats_text: str, + budget: float, + label: str, + min_samples: int, + min_symbols: int, + min_claimed: int, +) -> tuple[bool, list[str], Profile | None]: + problems: list[str] = [] + profile = parse_report(report) + + if profile.root < min_samples: + problems.append( + f"only {profile.root} root samples (need >= {min_samples}). " + f"`sample` did not attach for long enough, or the program exited " + f"first — a low share here would mean nothing." + ) + if len(profile.symbols) < min_symbols: + problems.append( + f"only {len(profile.symbols)} distinct perry-runtime symbols " + f"(need >= {min_symbols}). Either the binary is stripped — build " + f"with PERRY_DEBUG_SYMBOLS=1 — or this is a narrow loop, not the " + f"broad workload this budget is about." + ) + + stats = parse_stats(stats_text) + if stats is None: + problems.append( + "no `[tls-hot]` line: run the program with PERRY_TLS_HOT_STATS=1. " + "Without it there is no evidence the hot-slot cache was even live." + ) + else: + if stats["direct_tsd"] != 1: + problems.append( + "direct_tsd=0: `hot()` fell back to `_tlv_get_addr`, so the " + "cache is inert and this measurement is about a configuration " + "nobody ships." + ) + if stats["claimed"] < min_claimed: + problems.append( + f"claimed={stats['claimed']} generic slots (need >= " + f"{min_claimed}). This subject barely touches thread-locals " + f"outside the sixteen named fields, which makes it the wrong " + f"subject: it would pass this budget forever while the real " + f"cost grew elsewhere." + ) + if stats["claimed"] >= stats["capacity"]: + problems.append( + f"claimed={stats['claimed']} reached capacity " + f"{stats['capacity']}: declarations past the ceiling silently " + f"fall back to `_tlv_get_addr`." + ) + + over_budget = profile.share > budget + if over_budget: + problems.append( + f"{TARGET} is {profile.share:.1f}% of {label} " + f"({profile.target}/{profile.root} samples), budget {budget:.1f}%." + ) + return (not problems), problems, profile + + +def self_test() -> int: + """Every rejection above, driven in both directions.""" + + def report(tlv: int, root: int, symbols: int) -> str: + lines = ["Call graph:", f" {root} start (in p) + 1 [0x1]"] + for i in range(symbols): + lines.append( + f" + {max(root // max(symbols, 1), 1)} " + f"_RNvNtCs_13perry_runtime3fn{i} (in p) + 1 [0x2]" + ) + lines.append(f" + {tlv} {TARGET} (in libdyld.dylib) + 4 [0x3]") + return "\n".join(lines) + + good_stats = "[tls-hot] claimed=60 published=55 capacity=768 direct_tsd=1" + failures = [] + + ok, _, prof = check(report(100, 10000, 40), good_stats, 5.0, "x", 2000, 20, 20) + if not ok: + failures.append("a clean 1.0% run was rejected") + if prof is None or abs(prof.share - 1.0) > 0.01: + failures.append("share arithmetic is wrong") + + cases = [ + ("over budget", report(2000, 10000, 40), good_stats, 5.0, 2000, 20, 20), + ("too few samples", report(10, 500, 40), good_stats, 5.0, 2000, 20, 20), + ("stripped/narrow", report(10, 10000, 3), good_stats, 5.0, 2000, 20, 20), + ("no stats line", report(10, 10000, 40), "", 5.0, 2000, 20, 20), + ( + "inert cache", + report(10, 10000, 40), + "[tls-hot] claimed=60 published=55 capacity=768 direct_tsd=0", + 5.0, + 2000, + 20, + 20, + ), + ( + "covered subject", + report(10, 10000, 40), + "[tls-hot] claimed=3 published=3 capacity=768 direct_tsd=1", + 5.0, + 2000, + 20, + 20, + ), + ( + "slots exhausted", + report(10, 10000, 40), + "[tls-hot] claimed=768 published=700 capacity=768 direct_tsd=1", + 5.0, + 2000, + 20, + 20, + ), + ] + for name, rep, stats, budget, ms, msym, mc in cases: + ok, _, _ = check(rep, stats, budget, "x", ms, msym, mc) + if ok: + failures.append(f"{name}: accepted a run it must reject") + + for f in failures: + print(f"SELF-TEST FAILED: {f}", file=sys.stderr) + if failures: + return 1 + print(f"self-test: the checker rejects all {len(cases)} failure modes") + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--report", help="a `sample` report") + ap.add_argument("--stats", help="stderr of the PERRY_TLS_HOT_STATS=1 run") + ap.add_argument("--budget", type=float, default=5.0, help="max %% of root samples") + ap.add_argument("--label", default="the program") + ap.add_argument("--min-samples", type=int, default=2000) + ap.add_argument("--min-symbols", type=int, default=20) + ap.add_argument( + "--min-claimed", + type=int, + default=20, + help="floor on generic slots the subject must actually resolve", + ) + ap.add_argument("--self-test", action="store_true") + args = ap.parse_args() + + if args.self_test: + return self_test() + if not args.report or not args.stats: + ap.error("--report and --stats are required") + + report = open(args.report).read() + stats = open(args.stats).read() + ok, problems, profile = check( + report, + stats, + args.budget, + args.label, + args.min_samples, + args.min_symbols, + args.min_claimed, + ) + assert profile is not None + print(f"=== {args.label}") + print(f" root samples : {profile.root}") + print(f" {TARGET:<13}: {profile.target} ({profile.share:.1f}%) budget {args.budget:.1f}%") + print(f" stats : {parse_stats(stats)}") + if profile.callers: + print(" remaining callers:") + for sym, n in profile.callers.most_common(8): + print(f" {n:6d} {sym}") + if ok: + print(f" VERDICT : PASS") + return 0 + print(" VERDICT : FAIL") + for p in problems: + print(f" - {p}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tls_budget_gate.sh b/scripts/tls_budget_gate.sh new file mode 100755 index 0000000000..d1ea647d06 --- /dev/null +++ b/scripts/tls_budget_gate.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# The #7469 `_tlv_get_addr` budget gate: keeps Darwin's thread-local access +# cost from creeping back into the runtime a fourth time. +# +# WHY THIS EXISTS +# +# On Darwin a `thread_local!` access is an out-of-line call to `_tlv_get_addr`. +# `crates/perry-runtime/src/tls_hot.rs` has removed that cost three times — +# 0% of `churn_alloc` after #7565, then 8-9%, then 11% on `interp`/`retain`, +# then 20.5% of `asyncpipe` — and nothing ever noticed the creep, because +# nothing measured it. The mechanism was never the problem; the absence of a +# gate was. +# +# THIS SCRIPT IS DESIGNED TO BE ABLE TO FAIL, checked against CLAUDE.md's +# "four ways a gate can be unable to fail": +# +# 1. no `continue-on-error`, no `|| true`; `set -euo pipefail`, and the +# final line is a bare `exit "$rc"`. +# 2. NOT wired into branch protection by the change that adds it: a new gate +# has never been green, so promoting it immediately blocks every open PR +# (CLAUDE.md's corollary). That is a maintainer action after the first +# observed green run on `main` — and per the corollary, taking it is not +# optional follow-through. +# 3. the workflow's `concurrency` block cancels `pull_request` runs only. +# 4. THE SUBJECT MUST BE THE UNCOVERED ONE. This is the specific vacuity +# this gate exists to avoid. Measuring `churn_alloc` would pass forever +# while the real cost grew, because churn's thread-locals are exactly the +# sixteen the named-field cache was curated for — a gate green because +# its subject never ran. So the subjects are `asyncpipe` (Map/Set +# registries, buffer brands, descriptor state, template literals, async) +# and `interp` (inline-cache misses, field lookup, arguments objects), +# and `scripts/tls_budget_check.py` — the actual verdict logic — REFUSES +# to return a pass unless the run proves it exercised paths outside those +# sixteen: `PERRY_TLS_HOT_STATS=1` must report `direct_tsd=1` (else the +# cache is inert and a low share means the program resolved nothing) and +# `claimed` above a floor no allocation microbenchmark can clear. +# `--self-test` drives all seven of its rejections and runs on every PR. +# +# SABOTAGE CHECK (the proof this can go red, re-runnable by hand) +# +# Revert one hot declaration to a raw `thread_local!`: +# +# sed -i '' 's/^crate::perry_thread_local! {/thread_local! {/' \ +# crates/perry-runtime/src/buffer/header.rs +# +# Rebuild and re-run this gate. `asyncpipe`'s share goes from ~1% to ~9%, +# `interp`'s from ~1% to ~6%, and both fail. Restoring the macro restores +# the pass. Recorded in the PR that introduced this file. +# +# USAGE +# +# scripts/tls_budget_gate.sh [] + +set -euo pipefail + +PERRY_BIN="${1:?usage: tls_budget_gate.sh [out-dir]}" +OUT_DIR="${2:-$(mktemp -d)}" +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +FIXTURES="$REPO_ROOT/benchmarks/tls-budget" + +mkdir -p "$OUT_DIR" +PERRY_BIN="$(cd "$(dirname "$PERRY_BIN")" && pwd)/$(basename "$PERRY_BIN")" + +if [[ "$(uname -s)" != "Darwin" ]]; then + echo "tls-budget: _tlv_get_addr is a Darwin TLS artefact; nothing to measure on $(uname -s)." >&2 + exit 0 +fi + +# name | expected stdout | budget (% of root samples) +PROGRAMS=( + "asyncpipe|39197275 4718978 6 39197275 0|5.0" + "interp|17088400|3.0" +) + +SAMPLE_SECONDS="${TLS_BUDGET_SAMPLE_SECONDS:-8}" +rc=0 + +for entry in "${PROGRAMS[@]}"; do + IFS='|' read -r name expected budget <<<"$entry" + src="$FIXTURES/$name.ts" + exe="$OUT_DIR/$name" + + echo "--- compiling $name" + # Debug symbols so `sample` can name frames: an unsymbolicated profile + # reports 0% for everything, which would read as a pass. + # + # Explicitly checked rather than left to `set -e`: a compiler that cannot + # run is the one failure that must never be mistaken for "nothing to + # measure", and a subshell's status is easy to lose behind a pipe. + if ! ( cd "$OUT_DIR" && PERRY_DEBUG_SYMBOLS=1 "$PERRY_BIN" "$src" -o "$exe" >/dev/null ); then + echo "tls-budget: compiling $name failed" >&2 + rc=1 + continue + fi + if [[ ! -x "$exe" ]]; then + echo "tls-budget: compiling $name produced no executable" >&2 + rc=1 + continue + fi + + # Correctness BEFORE timing: a program that prints the wrong answer is not + # a faster program. + actual="$("$exe")" + if [[ "$actual" != "$expected" ]]; then + echo "tls-budget: $name printed '$actual', expected '$expected'" >&2 + rc=1 + continue + fi + + echo "--- profiling $name (${SAMPLE_SECONDS}s)" + PERRY_TLS_HOT_STATS=1 "$exe" >/dev/null 2>"$OUT_DIR/$name.stats" & + pid=$! + sleep 1 + sample "$pid" "$SAMPLE_SECONDS" -f "$OUT_DIR/$name.sample" >/dev/null 2>&1 || true + wait "$pid" + + if [[ ! -s "$OUT_DIR/$name.sample" ]]; then + echo "tls-budget: sample produced no report for $name" >&2 + rc=1 + continue + fi + + if ! python3 "$REPO_ROOT/scripts/tls_budget_check.py" \ + --report "$OUT_DIR/$name.sample" \ + --stats "$OUT_DIR/$name.stats" \ + --budget "$budget" \ + --label "$name"; then + rc=1 + fi +done + +exit "$rc"