diff --git a/CLAUDE.md b/CLAUDE.md index f847486c2f..e1579ae351 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Perry is a native TypeScript compiler written in Rust that compiles TypeScript source code directly to native executables. It uses SWC for TypeScript parsing and LLVM for code generation. -**Current Version:** 0.5.1312 +**Current Version:** 0.5.1313 ## TypeScript Parity Status diff --git a/Cargo.lock b/Cargo.lock index 2fbf36c3cb..0842a4470d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5547,7 +5547,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "perry" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "anyhow", "base64", @@ -5607,14 +5607,14 @@ dependencies = [ [[package]] name = "perry-api-manifest" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "serde", ] [[package]] name = "perry-audio-miniaudio" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "cc", "libc", @@ -5622,7 +5622,7 @@ dependencies = [ [[package]] name = "perry-codegen" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "anyhow", "inkwell", @@ -5639,7 +5639,7 @@ dependencies = [ [[package]] name = "perry-codegen-arkts" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "anyhow", "perry-hir", @@ -5647,7 +5647,7 @@ dependencies = [ [[package]] name = "perry-codegen-glance" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "anyhow", "perry-hir", @@ -5655,7 +5655,7 @@ dependencies = [ [[package]] name = "perry-codegen-js" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "anyhow", "perry-dispatch", @@ -5664,7 +5664,7 @@ dependencies = [ [[package]] name = "perry-codegen-swiftui" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "anyhow", "perry-hir", @@ -5672,7 +5672,7 @@ dependencies = [ [[package]] name = "perry-codegen-wasm" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "anyhow", "base64", @@ -5684,7 +5684,7 @@ dependencies = [ [[package]] name = "perry-codegen-wear-tiles" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "anyhow", "perry-hir", @@ -5692,7 +5692,7 @@ dependencies = [ [[package]] name = "perry-container-compose" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "anyhow", "async-trait", @@ -5721,14 +5721,14 @@ dependencies = [ [[package]] name = "perry-container-e2e" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "anyhow", ] [[package]] name = "perry-diagnostics" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "serde", "serde_json", @@ -5736,7 +5736,7 @@ dependencies = [ [[package]] name = "perry-dispatch" -version = "0.5.1312" +version = "0.5.1313" [[package]] name = "perry-doc-fixture-my-bindings" @@ -5747,7 +5747,7 @@ dependencies = [ [[package]] name = "perry-doc-tests" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "anyhow", "clap", @@ -5762,7 +5762,7 @@ dependencies = [ [[package]] name = "perry-ext-ads" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "block2", "objc2", @@ -5772,7 +5772,7 @@ dependencies = [ [[package]] name = "perry-ext-argon2" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "argon2", "perry-ffi", @@ -5780,7 +5780,7 @@ dependencies = [ [[package]] name = "perry-ext-axios" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "perry-ffi", "reqwest", @@ -5789,7 +5789,7 @@ dependencies = [ [[package]] name = "perry-ext-bcrypt" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "bcrypt", "perry-ffi", @@ -5797,7 +5797,7 @@ dependencies = [ [[package]] name = "perry-ext-better-sqlite3" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "perry-ffi", "rusqlite", @@ -5805,7 +5805,7 @@ dependencies = [ [[package]] name = "perry-ext-cheerio" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "perry-ffi", "scraper", @@ -5813,7 +5813,7 @@ dependencies = [ [[package]] name = "perry-ext-commander" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "perry-ffi", "perry-runtime", @@ -5821,7 +5821,7 @@ dependencies = [ [[package]] name = "perry-ext-cron" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "chrono", "cron", @@ -5831,7 +5831,7 @@ dependencies = [ [[package]] name = "perry-ext-dayjs" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "chrono", "perry-ffi", @@ -5839,7 +5839,7 @@ dependencies = [ [[package]] name = "perry-ext-decimal" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "perry-ffi", "rust_decimal", @@ -5847,7 +5847,7 @@ dependencies = [ [[package]] name = "perry-ext-dotenv" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "perry-ffi", "serde_json", @@ -5855,7 +5855,7 @@ dependencies = [ [[package]] name = "perry-ext-ethers" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "perry-ffi", "rand 0.10.1", @@ -5863,7 +5863,7 @@ dependencies = [ [[package]] name = "perry-ext-events" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "perry-ffi", "perry-runtime", @@ -5871,14 +5871,14 @@ dependencies = [ [[package]] name = "perry-ext-exponential-backoff" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "perry-ffi", ] [[package]] name = "perry-ext-fastify" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "bytes", "http-body-util", @@ -5896,7 +5896,7 @@ dependencies = [ [[package]] name = "perry-ext-fetch" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "bytes", "lazy_static", @@ -5909,7 +5909,7 @@ dependencies = [ [[package]] name = "perry-ext-http" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "bytes", "h2", @@ -5933,7 +5933,7 @@ dependencies = [ [[package]] name = "perry-ext-ioredis" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "lazy_static", "perry-ffi", @@ -5943,7 +5943,7 @@ dependencies = [ [[package]] name = "perry-ext-jsonwebtoken" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "base64", "jsonwebtoken", @@ -5954,7 +5954,7 @@ dependencies = [ [[package]] name = "perry-ext-lru-cache" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "lru", "perry-ffi", @@ -5963,7 +5963,7 @@ dependencies = [ [[package]] name = "perry-ext-moment" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "chrono", "perry-ffi", @@ -5971,7 +5971,7 @@ dependencies = [ [[package]] name = "perry-ext-mongodb" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "bson", "futures-util", @@ -5983,7 +5983,7 @@ dependencies = [ [[package]] name = "perry-ext-mysql2" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "chrono", "perry-ffi", @@ -5993,7 +5993,7 @@ dependencies = [ [[package]] name = "perry-ext-nanoid" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "nanoid", "perry-ffi", @@ -6002,7 +6002,7 @@ dependencies = [ [[package]] name = "perry-ext-net" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "bytes", "perry-ffi", @@ -6015,7 +6015,7 @@ dependencies = [ [[package]] name = "perry-ext-node-forge" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "const-oid 0.9.6", "der 0.7.10", @@ -6034,7 +6034,7 @@ dependencies = [ [[package]] name = "perry-ext-nodemailer" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "lettre", "perry-ffi", @@ -6044,7 +6044,7 @@ dependencies = [ [[package]] name = "perry-ext-pdf" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "perry-ffi", "printpdf", @@ -6052,7 +6052,7 @@ dependencies = [ [[package]] name = "perry-ext-pg" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "perry-ffi", "sqlx", @@ -6061,7 +6061,7 @@ dependencies = [ [[package]] name = "perry-ext-ratelimit" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "governor", "perry-ffi", @@ -6069,7 +6069,7 @@ dependencies = [ [[package]] name = "perry-ext-sharp" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "fast_image_resize", "image", @@ -6079,14 +6079,14 @@ dependencies = [ [[package]] name = "perry-ext-slugify" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "perry-ffi", ] [[package]] name = "perry-ext-streams" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "lazy_static", "perry-ffi", @@ -6095,7 +6095,7 @@ dependencies = [ [[package]] name = "perry-ext-undici" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "perry-ffi", "perry-runtime", @@ -6104,7 +6104,7 @@ dependencies = [ [[package]] name = "perry-ext-uuid" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "perry-ffi", "uuid", @@ -6112,7 +6112,7 @@ dependencies = [ [[package]] name = "perry-ext-validator" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "perry-ffi", "regex", @@ -6122,7 +6122,7 @@ dependencies = [ [[package]] name = "perry-ext-ws" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "futures-util", "lazy_static", @@ -6135,7 +6135,7 @@ dependencies = [ [[package]] name = "perry-ext-zlib" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "brotli", "flate2", @@ -6145,7 +6145,7 @@ dependencies = [ [[package]] name = "perry-ffi" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "dashmap", "once_cell", @@ -6154,7 +6154,7 @@ dependencies = [ [[package]] name = "perry-hir" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "anyhow", "perry-api-manifest", @@ -6172,7 +6172,7 @@ dependencies = [ [[package]] name = "perry-parser" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "anyhow", "perry-diagnostics", @@ -6184,7 +6184,7 @@ dependencies = [ [[package]] name = "perry-runtime" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "anyhow", "base64", @@ -6226,14 +6226,14 @@ dependencies = [ [[package]] name = "perry-runtime-static" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "perry-runtime", ] [[package]] name = "perry-stdlib" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "aes 0.8.4", "aes 0.9.1", @@ -6328,14 +6328,14 @@ dependencies = [ [[package]] name = "perry-stdlib-static" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "perry-stdlib", ] [[package]] name = "perry-transform" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "anyhow", "perry-hir", @@ -6344,14 +6344,14 @@ dependencies = [ [[package]] name = "perry-ui" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "perry-ui-model", ] [[package]] name = "perry-ui-android" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "base64", "itoa", @@ -6368,7 +6368,7 @@ dependencies = [ [[package]] name = "perry-ui-geisterhand" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "rand 0.10.1", "serde", @@ -6378,7 +6378,7 @@ dependencies = [ [[package]] name = "perry-ui-gtk4" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "base64", "cairo-rs 0.22.0", @@ -6401,7 +6401,7 @@ dependencies = [ [[package]] name = "perry-ui-ios" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "base64", "block2", @@ -6417,7 +6417,7 @@ dependencies = [ [[package]] name = "perry-ui-macos" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "base64", "block2", @@ -6432,7 +6432,7 @@ dependencies = [ [[package]] name = "perry-ui-model" -version = "0.5.1312" +version = "0.5.1313" [[package]] name = "perry-ui-test" @@ -6443,11 +6443,11 @@ dependencies = [ [[package]] name = "perry-ui-testkit" -version = "0.5.1312" +version = "0.5.1313" [[package]] name = "perry-ui-tvos" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "base64", "block2", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "perry-ui-visionos" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "base64", "block2", @@ -6479,7 +6479,7 @@ dependencies = [ [[package]] name = "perry-ui-watchos" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "block2", "libc", @@ -6492,7 +6492,7 @@ dependencies = [ [[package]] name = "perry-ui-windows" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "base64", "libc", @@ -6509,14 +6509,14 @@ dependencies = [ [[package]] name = "perry-ui-windows-winui" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "perry-ui-windows", ] [[package]] name = "perry-updater" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "anyhow", "base64", @@ -6532,7 +6532,7 @@ dependencies = [ [[package]] name = "perry-wasm-host" -version = "0.5.1312" +version = "0.5.1313" dependencies = [ "wasmi", ] diff --git a/Cargo.toml b/Cargo.toml index f2e0189cc2..dec544224b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -315,7 +315,7 @@ codegen-units = 16 codegen-units = 16 [workspace.package] -version = "0.5.1312" +version = "0.5.1313" edition = "2021" license = "MIT" repository = "https://github.com/PerryTS/perry" diff --git a/changelog.d/7539-json-tape-side-allocation.md b/changelog.d/7539-json-tape-side-allocation.md new file mode 100644 index 0000000000..3ab5ec4141 --- /dev/null +++ b/changelog.d/7539-json-tape-side-allocation.md @@ -0,0 +1,51 @@ +### Performance + +- **The JSON tape no longer lands in the old generation, so `JSON.parse` stops firing `old_gen_bytes` full collections (#7539).** Split out of #7478's decomposition; with #7537's scan flip landed, this was the whole remaining `field_access` gap. + + **Mechanism, confirmed before it was fixed.** A `LazyArrayHeader` was allocated as ONE arena object with its tape copied inline after the header, so the whole allocation was as large as the tape — ~2.4 MB for the 10 000-record fixture (200 002 `TapeEntry`s at 12 bytes). That is 150× `LARGE_OBJECT_THRESHOLD_BYTES` (16 KB), so `arena_alloc_gc`'s large-object arm routed it straight into the OLD generation and stamped `GC_FLAG_TENURED` on it. Old-generation bytes are reclaimable only by a FULL collection, so a tape that dies at the end of its loop iteration still accumulated at ~2.4 MB per parse until `old_reclaim_pressure_due` fired (48 MB absolute, or 32 MB of growth). + + Being *large* is not evidence of being *old*. The header was born tenured on the strength of its size alone, which handed the collector's cheapest question — "did this die in the nursery?" — to its most expensive answer. + + `PERRY_GC_TRACE=1` over the 53 parses of `benchmarks/json_polyglot/bench_field_access.ts` at the parent commit: + + | arm | cycles | full | `old_gen_bytes`-triggered | peak old-gen | + |---|--:|--:|--:|--:| + | tape + gen-GC (default) | 19 | 9 | **6** | 43.9 MB | + | `PERRY_JSON_TAPE=0` + gen-GC | 14 | 5 | 2 | 47.7 MB | + | tape + `PERRY_GEN_GC=0` | 31 | 31 | **0** | 14.1 MB | + + The cleanest attribution is `bench.ts` (roundtrip), which never materialises anything: its nursery peaks at **4.1 MB** while the old generation peaks at **39.6 MB** and fires 5 `old_gen_bytes` fulls, identically under both collectors. In that program there is nothing in the old generation *but* the tape. That measurement is what promoted the issue's hypothesis to a cause — and it also ruled out the RSS-pressure theory the headline numbers first suggested: `evacuation_policy` reports `not_evaluated` on every cycle of every arm, and evacuation moved 0 bytes. + + **The fix.** The tape moves out of the GC heap into a `json_tape_store` side allocation, which the header owns. It qualifies on every test already applied to `Map`/`Set` entry buffers: it is **pointer-free by construction** (`TapeEntry` is `{ offset: u32, kind: u8, link: u32 }` — the struct's alignment is 4, so on a 64-bit target no field it has can hold a pointer, and the region has exactly one writer), **uniquely owned** by one header, and immutable and immovable after construction. So it never needs marking, scanning, copying, or rewriting. + + On top of the collector-driven lifetime, the owner disowns its tape **deterministically**: the instant `force_materialize_lazy` installs `materialized`, every subsequent read goes through the `ArrayHeader` and the tape is provably garbage, so it is freed right there with no collector involved. That is the path `field_access` takes — #7537 flips the scan to the batch parser after `scan_flip_threshold` elements, a few hundred of 10 000 — which is why the result does not depend on GC timing for the workload that motivated it. Every site that sets `materialized` now goes through one `install_materialized` helper so the release cannot drift away from the install. + + **The header stays exactly where it was, and that is load-bearing.** The first version of this change let the shrunken ~88-byte header fall into the nursery, which made it MOVABLE for the first time in its life — its multi-megabyte inline tape had always parked it in the old generation. Callers outside `json_tape` were written against that: `json::stringify_api::try_stringify_lazy_array` reads `blob_bytes` off a raw header and then allocates the result string, and the array accessors pass raw headers into `force_materialize_lazy`. The copying minor promptly relocated the header out from under them, and `field_access` went non-deterministic — `JSON.stringify(parsed)` returned a JSON string of NUL bytes (a stale `blob_str` read through a moved-from header) on 3 of 60 iterations, while every element value stayed correct. So the header is now allocated old-gen and born tenured *explicitly* (`arena_alloc_gc_old_born_tenured`), stating the invariant instead of relying on the tape to imply it, and `GC_TYPE_LAZY_ARRAY` is marked non-movable so old-page defrag can never change it. That also means the tape registry needs no move hook and no copied-minor from-space pass. The cost is ~96 bytes of old generation per parse instead of ~2.4 MB. + + The sparse element cache and its bitmap move to the same generation for the same reason. An old-gen header with a nursery cache is a mix nothing covers: a minor treats the old header as a black leaf, so it never visits the `GcRewriteDescriptorKind::LazyArray` descriptor — the only thing that can read the cache — while the cache block is itself a GC leaf whose contents no walker scans. That combination lost element identity (`parsed[i] === parsed[i]`) across a copying minor. It could not arise before: a big array's cache was already born old, and a small array's header was born young alongside its cache. + + **One accounting trap, and why it is not one.** Tape bytes stay in `external_side_live_bytes()`, which every `old_reclaim_pressure_due` call site adds to old-generation pressure. That looks like it would re-create the pathology, and an intermediate version of this change removed it for exactly that reason. It was the wrong call: the old cost was *dead* tape sitting in the old generation as unreclaimable arena capacity, and `field_access` now disowns each tape at materialization, so the term never accumulates there — it holds one live tape at a time. `roundtrip` genuinely retains its tape until the lazy array dies, and those bytes must be able to escalate the reclaim that frees them, exactly like a dead Map's entries buffer; it keeps the bounded cadence it has today. + + **Result.** Pinned quiet host (M1 mini, `taskpolicy -t 0 -l 0`, 11 runs, load ≤ 2), same binaries end to end: + + | `field_access` | median | σ | peak RSS | + |---|--:|--:|--:| + | before, default | 1957 ms | 143.8 | 196 MB | + | **after, default** | **1809 ms** | **17.3** | **155 MB** | + | before, `PERRY_GEN_GC=0` | 1751 ms | 4.2 | 76 MB | + | after, `PERRY_GEN_GC=0` | 1604 ms | 3.0 | 76 MB | + | after, `PERRY_JSON_TAPE=0` | 1758 ms | 117.2 | 168 MB | + + σ collapses 8.3×, which was the headline symptom, and RSS drops 41 MB. The decisive row is the last one: **turning the tape ON is no longer worse than turning it OFF** — the tape-off arm still carries σ 117.2 and 168 MB, so the residual variance and footprint are the generational collector's own behaviour on this workload and have nothing left to do with the tape. The GC trace agrees exactly: `field_access` goes from 19 cycles / 9 full / 6 `old_gen_bytes` to **14 / 5 / 2**, which is the `PERRY_JSON_TAPE=0` arm's profile to the cycle. + + `roundtrip` — the memcpy path this must not regress — improves: **201 → 193 ms** (σ 0.5 → 0.7), and its peak old-generation *in-use* falls 39.6 → 14.1 MB with reserved 58 → 26 MB. It keeps its 5 `old_gen_bytes` fulls, by design: it genuinely retains each tape until the lazy array dies, so those bytes should keep escalating the reclaim that frees them. + + The change also helps the mark-sweep arm (1751 → 1604 ms), which is worth noting because it is independent of GC pacing: an inline multi-megabyte allocation per parse cost real time regardless of collector. + + Not claimed: the ~200 ms and ~79 MB still between the default and `PERRY_GEN_GC=0` on this workload. The tape-off arm carries the same gap, so it is a separate term. + + **Coverage.** `gc/tests/lazy_tape_side_alloc.rs` pins the load-bearing claims: old-generation growth no longer scales with tape size (two blobs of the same element count whose tapes differ 3×, so the blob string and the sparse cache are held constant — measuring one parse against zero would only have proved that old-gen grew by *less* than the tape); the header is small but still old-gen, born tenured, and non-movable; a dead unmaterialized owner releases its tape on a full collection; and a minor neither releases nor moves a live owner's tape. `json_tape_tests.rs` pins the pointer-free claim structurally rather than by convention, and that a disowned tape reads as empty rather than as freed memory. `gc/tests/teardown.rs` covers thread exit. + + `PERRY_GC_VERIFY_EVACUATION=1` runs clean on both benchmarks and on a 60-iteration divergence probe, including with `PERRY_GC_FORCE_EVACUATE=1`. All 42 JSON/lazy `test-files/*.ts` that build under `PERRY_NO_AUTO_OPTIMIZE` match `node --experimental-strip-types` (26.5.1) byte for byte, and both benchmark checksums are identical to `main` across repeated runs. + + Two existing tests asserted that a copied minor *relocates* the lazy header — true only because their tiny fixtures made the header small enough to be nursery-resident, which production never was. They now assert the opposite, which is the real invariant, and keep their live half: the materialized array is young, does move, and its handle must still be refreshed. diff --git a/crates/perry-runtime/src/arena/allocators.rs b/crates/perry-runtime/src/arena/allocators.rs index e0273b0fa8..1270929140 100644 --- a/crates/perry-runtime/src/arena/allocators.rs +++ b/crates/perry-runtime/src/arena/allocators.rs @@ -154,6 +154,26 @@ pub fn arena_alloc_gc_old(size: usize, align: usize, obj_type: u8) -> *mut u8 { unsafe { raw.add(GC_HEADER_SIZE) } } +/// The old-gen + born-tenured shape `arena_alloc_gc` hands a LARGE object, for +/// a caller that wants it on size-independent grounds. +/// +/// #7539's `LazyArrayHeader` is the caller: it used to reach this arm by being +/// multi-megabyte (its tape was inline), and every caller outside `json_tape` +/// relies on the resulting header address being stable across allocations. +/// Moving the tape out shrank the header to ~88 bytes, which would have made +/// it nursery-resident and movable; asking for this shape explicitly keeps the +/// invariant those callers were already written against. +pub(crate) fn arena_alloc_gc_old_born_tenured(size: usize, align: usize, obj_type: u8) -> *mut u8 { + use crate::gc::{GcHeader, GC_FLAG_TENURED, GC_HEADER_SIZE}; + + let user_ptr = arena_alloc_gc_old(size, align, obj_type); + unsafe { + let header = user_ptr.sub(GC_HEADER_SIZE) as *mut GcHeader; + (*header).gc_flags |= GC_FLAG_TENURED; + } + user_ptr +} + pub(crate) fn arena_alloc_gc_old_excluding_pages( size: usize, align: usize, diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index 0d3cad7606..04027c5551 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -62,7 +62,9 @@ pub use allocators::{ arena_alloc, arena_alloc_gc, arena_alloc_gc_longlived, arena_alloc_gc_old, arena_alloc_longlived, arena_alloc_old, js_arena_alloc, }; -pub(crate) use allocators::{arena_alloc_gc_old_excluding_pages, arena_alloc_gc_survivor}; +pub(crate) use allocators::{ + arena_alloc_gc_old_born_tenured, arena_alloc_gc_old_excluding_pages, arena_alloc_gc_survivor, +}; // walk.rs #[cfg(feature = "diagnostics")] diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index 8545038e79..7ca1d3822c 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -817,15 +817,16 @@ pub extern "C" fn js_gc_init() { gc_init(); } -/// Release external Map/Set storage owned by the current thread. +/// Release external Map/Set/JSON-tape storage owned by the current thread. /// /// This is intentionally narrower than a general heap teardown: the arena -/// headers remain owned by the arena, while the collection registries own the -/// separately allocated buffers. The operation is idempotent and is called +/// headers remain owned by the arena, while the side-allocation registries own +/// the separately allocated buffers. The operation is idempotent and is called /// only once no more JavaScript work can run on this thread. #[no_mangle] pub extern "C" fn js_gc_release_current_thread_collection_side_allocations() { crate::map::release_current_thread_map_side_allocations(); + crate::json_tape_store::release_current_thread_lazy_tapes(); crate::set::release_current_thread_set_side_allocations(); } diff --git a/crates/perry-runtime/src/gc/oldgen.rs b/crates/perry-runtime/src/gc/oldgen.rs index 327f837155..731f2385a4 100644 --- a/crates/perry-runtime/src/gc/oldgen.rs +++ b/crates/perry-runtime/src/gc/oldgen.rs @@ -472,6 +472,34 @@ pub(super) fn sweep() -> u64 { sweep_with_age_bump(false).freed_bytes } +/// #7539: is the lazy JSON array at `addr` provably dead at sweep entry? +/// +/// Same rule `map.rs` applies to a registered Map: unmarked ∧ not pinned ∧ not +/// forwarded, and — for a MINOR trace, which never traces the old generation — +/// additionally not tenured and physically in the nursery. +unsafe fn registered_lazy_array_is_dead_post_trace(addr: usize, full_trace: bool) -> bool { + let Some(header) = crate::value::addr_class::try_read_gc_header(addr) else { + return false; + }; + if header.obj_type != GC_TYPE_LAZY_ARRAY { + return false; + } + let flags = header.gc_flags; + if flags & (GC_FLAG_MARKED | GC_FLAG_PINNED | GC_FLAG_FORWARDED) != 0 { + return false; + } + if full_trace { + return true; + } + if flags & GC_FLAG_TENURED != 0 { + return false; + } + matches!( + crate::arena::classify_heap_generation(addr), + crate::arena::HeapGeneration::Nursery + ) +} + pub(super) fn sweep_malloc_objects() -> u64 { let mut state = MallocSweepCycleState::new(true); state.finish_unbounded() @@ -1055,6 +1083,7 @@ pub(super) struct IncrementalSweepState { dead_sets: Vec, dead_buffers: Vec, dead_typed_arrays: Vec, + dead_lazy_arrays: Vec, malloc: MallocSweepCycleState, arena: ArenaSweepObjectsState, cleanup: Option, @@ -1077,6 +1106,7 @@ impl IncrementalSweepState { dead_sets: Vec::new(), dead_buffers: Vec::new(), dead_typed_arrays: Vec::new(), + dead_lazy_arrays: Vec::new(), malloc: MallocSweepCycleState::new(sweep_malloc), arena: ArenaSweepObjectsState::new( do_age_bump, @@ -1108,10 +1138,18 @@ impl IncrementalSweepState { self.dead_buffers = crate::buffer::collect_dead_registered_buffers_post_trace(full_trace); self.dead_typed_arrays = crate::typedarray::collect_dead_registered_typed_arrays_post_trace(full_trace); + // #7539: lazy JSON arrays own their tape bytes outside the GC heap. + // The copying minor has its own from-space pass; this covers the + // non-copying cycles, including a dead owner sitting in the ACTIVE + // nursery block that no sweeper ever object-walks. + self.dead_lazy_arrays = crate::json_tape_store::collect_owners(&|addr| unsafe { + registered_lazy_array_is_dead_post_trace(addr, full_trace) + }); if !self.dead_maps.is_empty() || !self.dead_sets.is_empty() || !self.dead_buffers.is_empty() || !self.dead_typed_arrays.is_empty() + || !self.dead_lazy_arrays.is_empty() { self.subphase = SweepCycleSubphase::CollectionSideBuffers; } @@ -1131,6 +1169,8 @@ impl IncrementalSweepState { crate::buffer::finalize_collected_dead_buffer(addr); } else if let Some(addr) = self.dead_typed_arrays.pop() { crate::typedarray::finalize_collected_dead_typed_array(addr); + } else if let Some(addr) = self.dead_lazy_arrays.pop() { + crate::json_tape_store::release(addr); } else { self.subphase = SweepCycleSubphase::Malloc; break; @@ -1141,6 +1181,7 @@ impl IncrementalSweepState { && self.dead_sets.is_empty() && self.dead_buffers.is_empty() && self.dead_typed_arrays.is_empty() + && self.dead_lazy_arrays.is_empty() { self.subphase = SweepCycleSubphase::Malloc; } diff --git a/crates/perry-runtime/src/gc/tests/alloc.rs b/crates/perry-runtime/src/gc/tests/alloc.rs index fa5b1c1346..b2cf273304 100644 --- a/crates/perry-runtime/src/gc/tests/alloc.rs +++ b/crates/perry-runtime/src/gc/tests/alloc.rs @@ -481,13 +481,21 @@ fn test_gc_type_metadata_covers_all_declared_types() { arena_walkable: true, rewrite_descriptor_kind: GcRewriteDescriptorKind::LazyArray, layout_slot_kind: GcLayoutSlotKind::None, - movable: true, - external_byte_policy: GcExternalBytePolicy::InlinePayload, + // #7539: NOT movable. The tape registry is keyed by the header + // address, and callers outside `json_tape` hold raw header + // pointers across allocations. + movable: false, + // #7539: the tape is a `json_tape_store` side allocation, not + // inline payload. Inline, it made the header as large as the tape + // (~2.4 MB on a 10k-record blob), which `arena_alloc_gc` routed + // into the old generation with GC_FLAG_TENURED — reclaimable only + // by a FULL collection. + external_byte_policy: GcExternalBytePolicy::SideAllocation, large_object_policy: GcLargeObjectPolicy::OldArenaWhenOverThreshold, pointer_free: false, move_hook_kind: GcMoveHookKind::None, rewrite_hook_kind: GcRewriteHookKind::None, - finalize_hook_kind: GcFinalizeHookKind::None, + finalize_hook_kind: GcFinalizeHookKind::LazyArrayTape, }, GcTypeInfo { type_id: GC_TYPE_BUFFER, diff --git a/crates/perry-runtime/src/gc/tests/lazy_tape_side_alloc.rs b/crates/perry-runtime/src/gc/tests/lazy_tape_side_alloc.rs new file mode 100644 index 0000000000..dee22b81a6 --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/lazy_tape_side_alloc.rs @@ -0,0 +1,262 @@ +//! The JSON tape is a side allocation, not old-generation arena bytes (#7539). +//! +//! A `LazyArrayHeader` used to carry its tape INLINE, so the whole allocation +//! was as large as the tape — ~2.4 MB for the 10 k-record `field_access` +//! fixture. That is over `LARGE_OBJECT_THRESHOLD_BYTES` (16 KB), so +//! `arena_alloc_gc` routed it into the OLD generation and stamped +//! `GC_FLAG_TENURED` on it, and old-gen bytes are reclaimable only by a FULL +//! collection. A tape that dies at the end of its loop iteration therefore +//! accumulated at ~2.4 MB per parse until `old_reclaim_pressure_due` fired. +//! +//! Measured at the parent commit with `PERRY_GC_TRACE=1` over 53 parses of +//! `benchmarks/json_polyglot/bench_field_access.ts`: 19 collections, 9 full, +//! **6 of those triggered by `old_gen_bytes`**. `bench.ts` (roundtrip, which +//! never materialises) is the cleanest attribution — its nursery peaks at +//! 4.1 MB while the OLD generation peaks at **39.6 MB** and fires 5 +//! `old_gen_bytes` fulls. In that program the old generation IS the tape. +//! +//! These tests pin the claims the fix rests on: +//! +//! 1. old-generation growth no longer SCALES with tape size; +//! 2. the header stays small — and stays OLD-GEN and immovable, which is the +//! contract every caller outside `json_tape` was already written against; +//! 3. the tape is freed the instant `materialized` is installed, with no +//! collector involvement — the path `field_access` takes after #7537; +//! 4. the tape is freed by a full collection when its owner dies +//! unmaterialized (the `roundtrip` shape), and NOT by a minor. + +use super::super::*; +use super::support::*; + +fn build_lazy(input: &[u8]) -> *mut crate::json_tape::LazyArrayHeader { + let text = crate::string::js_string_from_bytes(input.as_ptr(), input.len() as u32); + crate::json_tape::with_built_tape(input, |tape| unsafe { + crate::json_tape::alloc_lazy_array( + tape, + 0, + crate::json_tape::count_array_length(tape, 0), + text, + ) + }) + .expect("valid JSON should build a tape") +} + +const ELEMENTS: u32 = 20_000; + +/// `[0,1,...,N-1]` — one tape entry per element. +fn flat_blob() -> Vec { + let mut blob = Vec::with_capacity(256 * 1024); + blob.push(b'['); + for i in 0..ELEMENTS { + if i > 0 { + blob.push(b','); + } + blob.extend_from_slice(i.to_string().as_bytes()); + } + blob.push(b']'); + blob +} + +/// `[[0],[1],...,[N-1]]` — same element COUNT (so the same sparse-cache size) +/// and nearly the same blob length, but three tape entries per element. +fn nested_blob() -> Vec { + let mut blob = Vec::with_capacity(256 * 1024); + blob.push(b'['); + for i in 0..ELEMENTS { + if i > 0 { + blob.push(b','); + } + blob.push(b'['); + blob.extend_from_slice(i.to_string().as_bytes()); + blob.push(b']'); + } + blob.push(b']'); + blob +} + +fn big_blob() -> Vec { + flat_blob() +} + +fn tape_bytes_of(blob: &[u8]) -> usize { + crate::json_tape::build_tape(blob) + .expect("valid JSON") + .entries + .len() + * std::mem::size_of::() +} + +/// The load-bearing claim: old-generation growth no longer SCALES with the +/// tape, because the tape is not a GC allocation at all. +/// +/// Measuring one parse against zero would only prove that old-gen grew by less +/// than the tape — but a parse legitimately puts other things there (the +/// retained blob string and the sparse element cache are both well over +/// `LARGE_OBJECT_THRESHOLD_BYTES` at this size). So compare two blobs with the +/// SAME element count, and therefore the same cache and near-identical blob +/// bytes, whose tapes differ by ~3×. Before the fix the extra tape entries +/// landed in old-gen one-for-one; now the difference is only the few extra +/// bracket characters in the blob. +#[test] +fn test_old_generation_growth_does_not_scale_with_tape_size() { + let _guard = GcTestIsolationGuard::new(); + let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + + let flat = flat_blob(); + let nested = nested_blob(); + let flat_tape = tape_bytes_of(&flat); + let nested_tape = tape_bytes_of(&nested); + let tape_delta = nested_tape - flat_tape; + assert!( + flat_tape > 4 * crate::gc::LARGE_OBJECT_THRESHOLD_BYTES + && tape_delta > 4 * crate::gc::LARGE_OBJECT_THRESHOLD_BYTES, + "test premise: both tapes ({flat_tape} B, {nested_tape} B) and their \ + difference must be well over the large-object threshold, or this \ + test proves nothing" + ); + let blob_delta = nested.len() - flat.len(); + + let before_flat = crate::arena::old_gen_in_use_bytes(); + let _flat_lazy = build_lazy(&flat); + let flat_growth = crate::arena::old_gen_in_use_bytes() - before_flat; + + let before_nested = crate::arena::old_gen_in_use_bytes(); + let _nested_lazy = build_lazy(&nested); + let nested_growth = crate::arena::old_gen_in_use_bytes() - before_nested; + + let growth_delta = nested_growth.saturating_sub(flat_growth); + assert!( + growth_delta < tape_delta / 2, + "old-gen growth tracked the tape: {tape_delta} B more tape produced \ + {growth_delta} B more old-gen (blob grew only {blob_delta} B, and \ + the sparse cache is identical at {ELEMENTS} elements)" + ); +} + +/// The header allocation no longer scales with the tape — but it stays in the +/// OLD generation and born tenured, exactly where a multi-megabyte inline-tape +/// header always landed. +/// +/// That is the load-bearing half of this test, not a leftover. `json_tape_store` +/// keys a tape by its owner's address, and every caller outside `json_tape` +/// holds raw `*mut LazyArrayHeader` across allocations — +/// `json::stringify_api::try_stringify_lazy_array` reads `blob_bytes` off a raw +/// header and then allocates the result string. Letting the shrunken header +/// fall into the nursery made it movable for the first time and the copying +/// minor relocated it out from under those callers: `field_access` went +/// non-deterministic, emitting a JSON string of NUL bytes for +/// `JSON.stringify(parsed)` on 3 of 60 iterations. If a future change routes +/// the header allocation back through `arena_alloc_gc`, this fails. +#[test] +fn test_lazy_header_is_small_but_stays_old_gen_and_immovable() { + let _guard = GcTestIsolationGuard::new(); + let blob = big_blob(); + let tape_bytes = tape_bytes_of(&blob); + assert!(tape_bytes > 4 * crate::gc::LARGE_OBJECT_THRESHOLD_BYTES); + + let lazy = build_lazy(&blob); + + assert!( + crate::arena::pointer_in_old_gen(lazy as usize), + "the header must stay old-gen: callers outside json_tape hold raw \ + header pointers across allocations" + ); + assert!( + !crate::gc::gc_type_is_movable(crate::gc::GC_TYPE_LAZY_ARRAY), + "a lazy array must not be movable — its tape is keyed by its address" + ); + unsafe { + let header = (lazy as *const u8).sub(GC_HEADER_SIZE) as *const GcHeader; + assert_ne!( + (*header).gc_flags & GC_FLAG_TENURED, + 0, + "the header must be born tenured, as the large-object arm made it" + ); + assert!( + ((*header).size as usize) < crate::gc::LARGE_OBJECT_THRESHOLD_BYTES, + "the header allocation must not scale with the tape" + ); + assert_eq!( + (*lazy).tape_len as usize, + tape_bytes / std::mem::size_of::() + ); + } + assert_eq!( + crate::json_tape_store::registered_bytes(), + tape_bytes, + "the tape bytes must be accounted to the side-allocation store" + ); +} + +/// A lazy array that dies UNMATERIALIZED must still give its tape back. This +/// is the `roundtrip` shape: parse, stringify off the retained blob, drop. +/// The owner is old-gen, so a FULL collection is what proves it dead — which +/// is also why tape bytes stay in `external_side_live_bytes()`: they have to +/// be able to escalate that reclaim, exactly like a dead Map's entries buffer. +#[test] +fn test_dead_unmaterialized_owner_releases_its_tape_on_a_full_collection() { + let _guard = GcTestIsolationGuard::new(); + let blob = big_blob(); + + let bytes_before = crate::json_tape_store::registered_bytes(); + let lazy = build_lazy(&blob); + assert!( + crate::json_tape_store::registered_bytes() > bytes_before, + "test premise: the lazy array owns tape bytes" + ); + // Deliberately NOT rooted: the header is unreachable garbage. + let _ = lazy; + + let _ = + gc_collect_full_mark_sweep_with_trigger(GcTriggerSnapshot::capture(GcTriggerKind::Direct)); + + assert_eq!( + crate::json_tape_store::registered_bytes(), + bytes_before, + "a full collection must release a dead lazy array's tape" + ); +} + +/// A minor must NOT release a live old-gen owner's tape, and must not move the +/// owner. Minors never trace the old generation, so "unmarked" says nothing +/// about an old header — treating one as dead there would free a tape out from +/// under a live lazy array. +#[test] +fn test_a_minor_neither_releases_nor_moves_a_live_owners_tape() { + let _guard = CopyingNurseryTestGuard::new(1); + let blob = big_blob(); + + let bytes_before = crate::json_tape_store::registered_bytes(); + let lazy = build_lazy(&blob); + let owned = crate::json_tape_store::registered_bytes() - bytes_before; + assert!(owned > 0, "test premise: the lazy array owns tape bytes"); + js_shadow_slot_set(0, ptr_bits(lazy as usize)); + + let trace = collect_minor_trace(GcTriggerKind::ArenaBytes); + assert!( + trace.copying_nursery.eligible, + "test premise: a COPYING minor must have run, or nothing was exercised" + ); + + assert_eq!( + crate::json_tape_store::registered_bytes() - bytes_before, + owned, + "a minor must not touch a live old-gen owner's tape" + ); + assert_eq!( + (js_shadow_slot_get(0) & POINTER_MASK) as usize, + lazy as usize, + "the owner must not have moved" + ); + unsafe { + let tape = crate::json_tape::LazyArrayHeader::tape_slice(lazy); + assert_eq!(tape[0].kind, crate::json_tape::KIND_ARR_START); + } + let arr = unsafe { crate::json_tape::force_materialize_lazy(lazy) }; + assert_eq!(unsafe { (*arr).length }, ELEMENTS); + assert_eq!( + crate::json_tape_store::registered_bytes(), + bytes_before, + "materializing must release the tape" + ); +} diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index 925260be2f..604a8cfb3f 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -21,6 +21,7 @@ mod host_safepoints; mod incremental_sweep_reclaim; mod inline_pointer_bearing_contract; mod layout_trace; +mod lazy_tape_side_alloc; mod oldgen; mod os_tag; mod root_words; diff --git a/crates/perry-runtime/src/gc/tests/runtime_roots/callback_scanners.rs b/crates/perry-runtime/src/gc/tests/runtime_roots/callback_scanners.rs index bf098d1360..c51a599c93 100644 --- a/crates/perry-runtime/src/gc/tests/runtime_roots/callback_scanners.rs +++ b/crates/perry-runtime/src/gc/tests/runtime_roots/callback_scanners.rs @@ -270,9 +270,19 @@ fn test_json_tape_lazy_get_header_handle_survives_copied_minor_gc() { JsonTapeSafepointHookGuard::new(crate::json_tape::JsonTapeSafepoint::LazyArrayRooted); let hdr = unsafe { test_alloc_lazy_json_array(input) }; let original_hdr = hook.fired_ptr(); - assert_ne!( + // #7539: the header is old-gen and immovable by construction, so a + // copied minor at the safepoint CANNOT relocate it — that is the + // property `try_stringify_lazy_array` and the array accessors rely on + // when they hold a raw header across an allocation. What must still be + // true is that `alloc_lazy_array` hands back the address the collector + // sees, i.e. the one its own rooted handle resolves to. + assert_eq!( hdr as usize, original_hdr, - "alloc_lazy_array should return the refreshed lazy header after copied-minor GC" + "the lazy header must not move across a copied-minor GC" + ); + assert!( + crate::arena::pointer_in_old_gen(hdr as usize), + "…because it is old-gen, which is what makes that guaranteed" ); hdr }; @@ -283,9 +293,9 @@ fn test_json_tape_lazy_get_header_handle_survives_copied_minor_gc() { let value = unsafe { crate::json_tape::lazy_get(hdr_handle.get_raw_mut_ptr(), 0) }; let original_hdr = hook.fired_ptr(); let hdr_after = hdr_handle.get_raw_mut_ptr::(); - assert_ne!( + assert_eq!( hdr_after as usize, original_hdr, - "lazy_get should refresh the rooted lazy header after copied-minor GC" + "the lazy header must not move across a copied-minor GC (#7539)" ); unsafe { let bitmap = (*hdr_after).materialized_bitmap; @@ -324,10 +334,12 @@ fn test_json_tape_lazy_get_records_its_cache_store_as_an_external_edge() { let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); register_runtime_handle_root_scanner_for_tests(); - // Born-old header: >16 KB of inline tape. That is the shape the #7538 - // workload had and the only one where the in-object/external distinction - // bites — a nursery header is traced directly and its descriptor reaches - // the cache without any remembered-set entry at all. + // Born-old header. That is the shape the #7538 workload had and the only + // one where the in-object/external distinction bites — a nursery header is + // traced directly and its descriptor reaches the cache without any + // remembered-set entry at all. #7539 moved the tape into a side allocation + // but deliberately kept the header in the old generation, so this premise + // still holds by construction rather than by the header being large. let elements = 4096; let mut input = String::with_capacity(elements * 8 + 2); input.push('['); @@ -344,14 +356,24 @@ fn test_json_tape_lazy_get_records_its_cache_store_as_an_external_edge() { "the lazy header must be born old, or this test exercises nothing" ); + // Read an element far enough into the array that its cache slot is + // GUARANTEED to sit on a different 4 KiB page from the header: slot 2048 + // is 16 KiB into the cache block. The test used to read element 7, whose + // slot happened to be pages away only because the header itself was + // multi-megabyte (the inline tape). Once #7539 shrank the header to + // ~88 bytes the cache landed immediately after it and the two shared a + // page, which would have made the containment assertion below fail for a + // reason that has nothing to do with the barrier under test. + // // No handle scope: the header is old-gen (asserted above), automatic // triggers are suppressed, and nothing here collects — so `hdr` cannot // move for the length of this test. - let first = unsafe { crate::json_tape::lazy_get(hdr, 7) }; + const PROBE: u32 = 2048; + let first = unsafe { crate::json_tape::lazy_get(hdr, PROBE) }; let element_addr = first.bits() & POINTER_MASK; assert_ne!( element_addr, 0, - "element 7 should materialize to a heap object" + "the probed element should materialize to a heap object" ); assert!( crate::arena::pointer_in_nursery(element_addr as usize), @@ -365,7 +387,7 @@ fn test_json_tape_lazy_get_records_its_cache_store_as_an_external_edge() { "cold lazy_get should allocate the sparse cache" ); ( - cache.add(7) as usize, + cache.add(PROBE as usize) as usize, header_from_user_ptr(hdr as *const u8) as usize, ) }; @@ -432,10 +454,13 @@ fn test_json_tape_force_materialize_sparse_cache_handles_survive_copied_minor_gc ); let original_arr = hook.fired_ptr(); let hdr_after = hdr_handle.get_raw_mut_ptr::(); - assert_ne!( + assert_eq!( hdr_after as usize, before_force_hdr, - "force materialization should refresh the rooted lazy header" + "the lazy header must not move across a copied-minor GC (#7539)" ); + // The MATERIALIZED ARRAY is young and does move — which is the handle + // refresh this test is really about, and the reason the header being + // stable does not make it vacuous. assert_ne!( arr as usize, original_arr, "force materialization should refresh the rooted array handle" diff --git a/crates/perry-runtime/src/gc/tests/teardown.rs b/crates/perry-runtime/src/gc/tests/teardown.rs index 406afdd19a..e145c6bc0e 100644 --- a/crates/perry-runtime/src/gc/tests/teardown.rs +++ b/crates/perry-runtime/src/gc/tests/teardown.rs @@ -166,3 +166,39 @@ fn map_set_owner_records_follow_growth() { assert!(set_after.0 - set_before.0 >= 1); assert!(set_after.1 - set_before.1 >= 64); } + +/// #7539: a lazy JSON array's tape is a side allocation too, so a thread that +/// exits still holding one must hand its bytes back. Runs entirely inside the +/// probe thread — `json_tape_store`'s registry and byte counter are +/// thread-local, so unlike the Map/Set counters above there is no cross-thread +/// window to widen the assertions for. +#[test] +fn lazy_tape_side_allocations_release_on_thread_exit() { + std::thread::spawn(|| { + let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + assert_eq!(crate::json_tape_store::registered_bytes(), 0); + + let input = b"[1,2,3,4,5,6,7,8]"; + let text = crate::string::js_string_from_bytes(input.as_ptr(), input.len() as u32); + let tape = crate::json_tape::build_tape(input).expect("valid JSON"); + let len = crate::json_tape::count_array_length(&tape.entries, 0); + // Left live and unmaterialized: only teardown can free this. + let _lazy = unsafe { crate::json_tape::alloc_lazy_array(&tape.entries, 0, len, text) }; + assert!( + crate::json_tape_store::registered_bytes() > 0, + "test premise: the thread exits owning tape bytes" + ); + + crate::gc::js_gc_release_current_thread_collection_side_allocations(); + assert_eq!( + crate::json_tape_store::registered_bytes(), + 0, + "thread teardown must release live tapes" + ); + // Idempotent, like the Map/Set drains above. + crate::gc::js_gc_release_current_thread_collection_side_allocations(); + assert_eq!(crate::json_tape_store::registered_bytes(), 0); + }) + .join() + .expect("lazy tape teardown probe thread should not panic"); +} diff --git a/crates/perry-runtime/src/gc/types.rs b/crates/perry-runtime/src/gc/types.rs index 227e361e9c..638932ad69 100644 --- a/crates/perry-runtime/src/gc/types.rs +++ b/crates/perry-runtime/src/gc/types.rs @@ -190,6 +190,9 @@ pub(crate) enum GcFinalizeHookKind { /// so a fresh error allocated at the recycled address doesn't inherit /// the dead error's codes/props. ErrorSideTables, + /// #7539: free a dead lazy JSON array's tape bytes, which + /// `json_tape_store` owns outside the GC heap. + LazyArrayTape, } #[allow(dead_code)] @@ -374,13 +377,26 @@ pub(super) static GC_TYPE_INFO_BY_ID: [Option; MALLOC_KIND_BUCKET_CO true, GcRewriteDescriptorKind::LazyArray, GcLayoutSlotKind::None, - true, - GcExternalBytePolicy::InlinePayload, + // NOT movable. `json_tape_store` keys a lazy array's tape by its + // header address, and every caller outside `json_tape` holds raw + // header pointers across allocations. The header is allocated old-gen + // and born tenured (`json_tape::alloc_lazy_header_bytes`), so nothing + // relocates it today; saying so here is what keeps old-page defrag + // from ever doing so. `true` was vacuous before #7539 anyway — the + // header was multi-megabyte and never left the old generation. + false, + // #7539: the tape is a `json_tape_store` side allocation now, not + // inline payload. Keeping it inline made the header ~2.4 MB on a + // 10 k-record blob, which `arena_alloc_gc` routed into the old + // generation with GC_FLAG_TENURED — reclaimable only by a FULL + // collection, so per-iteration-dead tapes drove `old_gen_bytes` + // fulls (6 of 9 fulls on the `field_access` fixture). + GcExternalBytePolicy::SideAllocation, GcLargeObjectPolicy::OldArenaWhenOverThreshold, false, GcMoveHookKind::None, GcRewriteHookKind::None, - GcFinalizeHookKind::None, + GcFinalizeHookKind::LazyArrayTape, )), Some(gc_type_info_entry( GC_TYPE_BUFFER, @@ -751,6 +767,9 @@ pub(crate) unsafe fn gc_type_finalize_unmarked_payload(obj_type: u8, user_ptr: * GcFinalizeHookKind::TypedArrayViewMeta => { crate::typedarray_view::clear_view_meta(user_ptr as usize); } + GcFinalizeHookKind::LazyArrayTape => { + crate::json_tape_store::release(user_ptr as usize); + } } } diff --git a/crates/perry-runtime/src/json_tape.rs b/crates/perry-runtime/src/json_tape.rs index fd2065ece7..ac3415224d 100644 --- a/crates/perry-runtime/src/json_tape.rs +++ b/crates/perry-runtime/src/json_tape.rs @@ -538,8 +538,10 @@ impl<'a, 'scope> TapeSource<'a, 'scope> { if hdr.is_null() || idx >= (*hdr).tape_len as usize { return None; } - let base = (hdr as *const u8).add(std::mem::size_of::()) - as *const TapeEntry; + let base = (*hdr).tape as *const TapeEntry; + if base.is_null() { + return None; + } Some(*base.add(idx)) } } @@ -961,8 +963,25 @@ pub struct LazyArrayHeader { pub magic: u32, /// Tape index where the root ARR_START sits. pub root_idx: u32, - /// Number of `TapeEntry`s that follow inline after this header. + /// Number of `TapeEntry`s reachable through [`Self::tape`]. pub tape_len: u32, + /// #7539: the tape's bytes, owned by `json_tape_store` rather than + /// allocated inline after this header. + /// + /// **Not a GC edge.** This points at a plain `std::alloc` buffer, never at + /// a managed object, so it is deliberately absent from the `LazyArray` + /// rewrite descriptor: nothing marks it, nothing rewrites it, and no write + /// barrier guards stores to it. The tape used to live inline, which made + /// the whole allocation ~2.4 MB on a 10 k-record blob — over + /// `LARGE_OBJECT_THRESHOLD_BYTES`, so `arena_alloc_gc` put it in the old + /// generation with `GC_FLAG_TENURED`, where only a FULL collection can + /// reclaim it. See `json_tape_store` for the measurement. + /// + /// Null once the tape is gone: either the blob had no entries, or + /// `force_materialize_lazy` disowned it after installing `materialized`. + /// Every reader checks `materialized.is_null()` before consulting the + /// tape, so a null here is only ever observed as "length 0". + pub tape: *mut TapeEntry, /// Owns-a-reference to the input `StringHeader`. GC must trace /// this to keep the blob alive while this lazy value is /// reachable. @@ -1026,7 +1045,6 @@ pub struct LazyArrayHeader { /// tokenization included). This counter is the missing signal; /// `scan_flip_threshold` is where it trips. pub sequential_streak: u32, - // Followed by `tape_len` `TapeEntry` elements inline. } // `cached_length` at offset 0 is a CODEGEN contract, not a layout preference: @@ -1095,12 +1113,17 @@ unsafe fn lazy_cached_count(hdr: *const LazyArrayHeader) -> u64 { } impl LazyArrayHeader { - /// Slice view over the inline tape bytes. Caller must keep the - /// header alive for the slice's lifetime. + /// Slice view over the tape bytes. Caller must keep the header alive for + /// the slice's lifetime. + /// + /// Empty once the tape has been disowned (`materialized` installed) — the + /// null check is what makes that state safe rather than a wild read. #[inline] pub unsafe fn tape_slice<'a>(this: *const LazyArrayHeader) -> &'a [TapeEntry] { - let base = - (this as *const u8).add(std::mem::size_of::()) as *const TapeEntry; + let base = (*this).tape; + if base.is_null() { + return &[]; + } std::slice::from_raw_parts(base, (*this).tape_len as usize) } @@ -1115,9 +1138,44 @@ impl LazyArrayHeader { } } -/// Arena-allocate a lazy array header with `tape_entries` copied -/// inline after the header. Returns the pointer that `JSON.parse` -/// hands back as a POINTER_TAG'd JSValue. +/// Arena-allocate a lazy array header owning `tape_entries` as a side +/// allocation. Returns the pointer that `JSON.parse` hands back as a +/// POINTER_TAG'd JSValue. +/// +/// #7539: the tape used to be copied INLINE after the header, making this one +/// allocation as large as the tape (~2.4 MB on a 10 k-record blob). That is +/// over `LARGE_OBJECT_THRESHOLD_BYTES`, so `arena_alloc_gc` routed it into the +/// old generation with `GC_FLAG_TENURED` and only a FULL collection could ever +/// reclaim it. The header is ~88 bytes now and is born in the nursery like any +/// other short-lived object; `json_tape_store` owns the tape bytes. +/// The header's own bytes. +/// +/// **Old generation, born tenured — and that is load-bearing, not incidental.** +/// Before #7539 the header carried its tape inline, so it was multi-megabyte +/// and `arena_alloc_gc`'s large-object arm put it here; every caller outside +/// this module has therefore always been free to hold a raw +/// `*mut LazyArrayHeader` across an allocation, and several do +/// (`json::stringify_api::try_stringify_lazy_array` reads `blob_bytes` off a +/// raw header and then allocates the result string; the array accessors pass +/// raw headers into `force_materialize_lazy`). +/// +/// Shrinking the header to ~88 bytes without pinning it here made it +/// nursery-resident and therefore MOVABLE for the first time, and the copying +/// minor promptly relocated it out from under those callers: `field_access` +/// went non-deterministic, emitting a JSON string of NUL bytes for +/// `JSON.stringify(parsed)` on 3 of 60 iterations (a stale `blob_str` read +/// through a moved-from header). Keeping the header exactly where it has +/// always been costs ~96 bytes of old generation per parse — the tape's +/// ~2.4 MB is what had to leave — and keeps that contract intact. +#[inline] +fn alloc_lazy_header_bytes() -> *mut u8 { + crate::arena::arena_alloc_gc_old_born_tenured( + std::mem::size_of::(), + 8, + crate::gc::GC_TYPE_LAZY_ARRAY, + ) +} + pub unsafe fn alloc_lazy_array( tape_entries: &[TapeEntry], root_idx: u32, @@ -1126,14 +1184,21 @@ pub unsafe fn alloc_lazy_array( ) -> *mut LazyArrayHeader { let scope = crate::gc::RuntimeHandleScope::new(); let blob_handle = scope.root_string_ptr(blob_str); - let tape_bytes = std::mem::size_of_val(tape_entries); - let total = std::mem::size_of::() + tape_bytes; - let raw = crate::arena::arena_alloc_gc(total, 8, crate::gc::GC_TYPE_LAZY_ARRAY); + // Detach the tape FIRST, while there is no header address to invalidate. + // The buffer itself is plain `std::alloc` memory — no arena accounting, no + // collection. `allocate` does account the bytes as external side pressure, + // which can trigger, but the only live thing we hold across it is + // `blob_handle`, which is rooted. + let (tape_ptr, tape_allocation) = crate::json_tape_store::allocate(tape_entries); + let raw = alloc_lazy_header_bytes(); let hdr = raw as *mut LazyArrayHeader; (*hdr).cached_length = cached_length; (*hdr).magic = LAZY_ARRAY_MAGIC; (*hdr).root_idx = root_idx; (*hdr).tape_len = tape_entries.len() as u32; + // GC_STORE_AUDIT(POINTER_FREE): side-allocated tape bytes, not a heap edge — + // no barrier, and deliberately absent from the LazyArray rewrite descriptor. + (*hdr).tape = tape_ptr; (*hdr).blob_str = blob_handle.get_raw_const_ptr::(); (*hdr).materialized = std::ptr::null_mut(); (*hdr).materialized_elements = std::ptr::null_mut(); @@ -1165,7 +1230,24 @@ pub unsafe fn alloc_lazy_array( // tape itself. if cached_length > 0 { let cache_bytes = (cached_length as usize) * std::mem::size_of::(); - let cache_raw = crate::arena::arena_alloc_gc(cache_bytes, 8, crate::gc::GC_TYPE_STRING); + // Old-gen, like the header (#7539). Keeping the whole lazy-array + // cluster in ONE generation keeps every edge out of it the shape + // #7538/#7546 built and validated the external-slot barrier for: + // old owner → old cache block → young element, recorded by + // `note_lazy_cache_slot` and consumed by the minor's dirty scan + // through the owner's descriptor. A nursery cache under an old-gen + // header is a mixed shape nothing covers — the minor treats the old + // header as a black leaf, so it never visits the descriptor that can + // read the cache, while the cache block itself is a GC leaf whose + // contents no walker scans. That combination lost element identity + // (`parsed[i] === parsed[i]`) across a copying minor. It could not + // occur before: a big array's cache was already born old, and a small + // array's header was born young along with its cache. + let cache_raw = crate::arena::arena_alloc_gc_old_born_tenured( + cache_bytes, + 8, + crate::gc::GC_TYPE_STRING, + ); // arena_alloc_gc can reuse slots from the free list whose // bytes still hold whatever the previous occupant wrote. // Zero explicitly — the cache invariant relies on the @@ -1183,7 +1265,14 @@ pub unsafe fn alloc_lazy_array( ); let bitmap_words = (cached_length as usize).div_ceil(64); let bitmap_bytes = bitmap_words * 8; - let bitmap_raw = crate::arena::arena_alloc_gc(bitmap_bytes, 8, crate::gc::GC_TYPE_STRING); + // Same generation as the header and cache — see above. The bitmap + // holds no heap edges, but keeping it with its cluster keeps the + // page-liveness bookkeeping uniform. + let bitmap_raw = crate::arena::arena_alloc_gc_old_born_tenured( + bitmap_bytes, + 8, + crate::gc::GC_TYPE_STRING, + ); std::ptr::write_bytes(bitmap_raw, 0, bitmap_bytes); let hdr = hdr_handle.get_raw_mut_ptr::(); (*hdr).materialized_bitmap = bitmap_raw as *mut u64; @@ -1193,11 +1282,60 @@ pub unsafe fn alloc_lazy_array( bitmap_raw as usize, ); } + // Register LAST, off the rooted handle. The header is old-gen and + // immovable (see `alloc_lazy_header_bytes`), so this address is stable for + // its whole life — which is what lets the registry be keyed by it with no + // move hook. let hdr = hdr_handle.get_raw_mut_ptr::(); - let tape_dst = (hdr as *mut u8).add(std::mem::size_of::()) as *mut TapeEntry; - // GC_STORE_AUDIT(POINTER_FREE): TapeEntry is offset/kind/link numerics, no heap edges. - std::ptr::copy_nonoverlapping(tape_entries.as_ptr(), tape_dst, tape_entries.len()); - hdr_handle.get_raw_mut_ptr::() + crate::json_tape_store::register(hdr as usize, tape_allocation); + hdr +} + +/// Install `arr_ptr` as this header's materialized array and disown the tape. +/// +/// Every site that sets `materialized` goes through here so the release can +/// never drift away from the install — the tape is garbage from the instant +/// `materialized` is non-null (`lazy_get`'s first fast path returns out of the +/// `ArrayHeader` and never consults the tape or the sparse cache again), and a +/// site that set the field directly would silently retain ~2.4 MB per parse +/// again. +/// +/// # Safety +/// +/// `hdr` must be a live `LazyArrayHeader` and `arr_ptr` a live `ArrayHeader`. +/// No `TapeSource::Lazy` read of this header may be in flight. +#[inline] +unsafe fn install_materialized(hdr: *mut LazyArrayHeader, arr_ptr: *mut crate::array::ArrayHeader) { + (*hdr).materialized = arr_ptr; + note_lazy_raw_slot( + hdr, + &(*hdr).materialized as *const _ as usize, + arr_ptr as usize, + ); + release_tape_after_materialize(hdr); +} + +/// Disown the tape once `materialized` is installed. +/// +/// After a full materialization every read goes through the `ArrayHeader`, so +/// the tape is provably garbage at this exact instant — no collector has to +/// prove it. Freeing here is what keeps `field_access` flat: #7537 flips a +/// scan to the batch parser after `scan_flip_threshold` elements, so the +/// ~2.4 MB tape becomes dead within the first few hundred of 10 000 reads and +/// is released immediately rather than waiting for the next full collection. +/// +/// # Safety +/// +/// `hdr` must be a live `LazyArrayHeader` whose `materialized` field is +/// already non-null, and no `TapeSource::Lazy` borrow of its tape may be live. +pub(crate) unsafe fn release_tape_after_materialize(hdr: *mut LazyArrayHeader) { + if hdr.is_null() || (*hdr).materialized.is_null() || (*hdr).tape.is_null() { + return; + } + crate::json_tape_store::release(hdr as usize); + // GC_STORE_AUDIT(POINTER_FREE): clears the side-allocation pointer after deregistration. + (*hdr).tape = std::ptr::null_mut(); + (*hdr).tape_len = 0; } #[inline] @@ -1582,12 +1720,7 @@ unsafe fn reparse_materialize( } } } - (*hdr).materialized = arr_ptr; - note_lazy_raw_slot( - hdr, - &(*hdr).materialized as *const _ as usize, - arr_ptr as usize, - ); + install_materialized(hdr, arr_ptr); } REPARSE_MATERIALIZATIONS.with(|c| c.set(c.get().wrapping_add(1))); (Some(arr_ptr), hdr) @@ -1647,12 +1780,7 @@ pub unsafe fn force_materialize_lazy(hdr: *mut LazyArrayHeader) -> *mut crate::a let arr_handle = scope.root_nanbox_u64(js.bits()); let arr_ptr = array_from_nanbox_handle(&arr_handle); let hdr = hdr_handle.get_raw_mut_ptr::(); - (*hdr).materialized = arr_ptr; - note_lazy_raw_slot( - hdr, - &(*hdr).materialized as *const _ as usize, - arr_ptr as usize, - ); + install_materialized(hdr, arr_ptr); return arr_ptr; } @@ -1670,12 +1798,7 @@ pub unsafe fn force_materialize_lazy(hdr: *mut LazyArrayHeader) -> *mut crate::a if root_entry.kind != KIND_ARR_START { let arr_ptr = array_from_nanbox_handle(&arr_handle); let hdr = hdr_handle.get_raw_mut_ptr::(); - (*hdr).materialized = arr_ptr; - note_lazy_raw_slot( - hdr, - &(*hdr).materialized as *const _ as usize, - arr_ptr as usize, - ); + install_materialized(hdr, arr_ptr); return arr_ptr; } let end = root_entry.link as usize; @@ -1725,12 +1848,7 @@ pub unsafe fn force_materialize_lazy(hdr: *mut LazyArrayHeader) -> *mut crate::a let arr_ptr = array_from_nanbox_handle(&arr_handle); (*arr_ptr).length = cached_length; let hdr = hdr_handle.get_raw_mut_ptr::(); - (*hdr).materialized = arr_ptr; - note_lazy_raw_slot( - hdr, - &(*hdr).materialized as *const _ as usize, - arr_ptr as usize, - ); + install_materialized(hdr, arr_ptr); arr_ptr } diff --git a/crates/perry-runtime/src/json_tape_store.rs b/crates/perry-runtime/src/json_tape_store.rs new file mode 100644 index 0000000000..dcaca0d4fc --- /dev/null +++ b/crates/perry-runtime/src/json_tape_store.rs @@ -0,0 +1,279 @@ +//! Owner of the JSON tape's backing bytes (#7539). +//! +//! A `LazyArrayHeader` used to be allocated as ONE arena object with its tape +//! copied inline after the header. For the 10 k-record `field_access` fixture +//! that is ~2.4 MB in a single allocation, which puts it over +//! `LARGE_OBJECT_THRESHOLD_BYTES` (16 KB), so `arena_alloc_gc` routed the whole +//! thing straight into the OLD generation with `GC_FLAG_TENURED` set. Old-gen +//! bytes are reclaimable only by a FULL collection, so a tape that dies at the +//! end of its loop iteration still accumulated at ~2.4 MB per parse until +//! `old_reclaim_pressure_due` fired (48 MB absolute / 32 MB growth). +//! +//! Measured at `origin/main` on the `bench_field_access.ts` fixture +//! (`PERRY_GC_TRACE=1`, 53 parses): 19 collections, 9 of them full, **6 of +//! those triggered by `old_gen_bytes`**. With `PERRY_JSON_TAPE=0` the same +//! program runs 14 collections with 5 fulls and only 2 `old_gen_bytes` +//! triggers. The cleanest attribution is `bench.ts` (roundtrip), which never +//! materialises anything: its nursery peaks at 4.1 MB while the OLD generation +//! peaks at **39.6 MB** and fires 5 `old_gen_bytes` fulls — in that program the +//! old generation *is* the tape. +//! +//! So the tape moves out of the GC heap entirely. It is a legitimate side +//! allocation by every test the runtime already applies to `Map`/`Set` entry +//! buffers: +//! +//! * **Pointer-free by construction.** `TapeEntry` is `{ offset: u32, kind: +//! u8, link: u32 }` — three integer fields, two of which are too narrow to +//! hold a 48-bit heap address, and the only writer of the region is one +//! `copy_nonoverlapping` from a `&[TapeEntry]`. It therefore never needs +//! scanning, marking, or rewriting. +//! * **Uniquely owned.** Exactly one `LazyArrayHeader` references a tape and +//! nothing else can, so ownership is exact and needs no tracing. +//! * **Immutable and immovable after construction.** +//! +//! Lifetime is the proven Map/Set side-allocation shape, keyed by the owning +//! header's address: +//! +//! * [`register`] at construction, [`release`] when the owner dies. +//! * `GcFinalizeHookKind::LazyArrayTape` covers the non-copying sweeps. +//! * [`finalize_dead_copied_minor_from_space_lazy_arrays`] covers the copying +//! minor, whose bulk from-space reset skips per-object finalizers. +//! * [`release_current_thread_lazy_tapes`] at thread teardown. +//! +//! There is deliberately no move hook and no copied-minor from-space pass: the +//! owning header stays OLD-GEN and immovable (see +//! `json_tape::alloc_lazy_header_bytes`), so its address is stable for its +//! whole life and it never appears in a from-space. Shrinking the header into +//! the nursery instead would have made it movable for the first time and broke +//! `json::stringify_api::try_stringify_lazy_array`, which reads `blob_bytes` +//! off a raw header and then allocates. +//! +//! On top of that the owner can disown its tape *deterministically*: once +//! `force_materialize_lazy` installs `materialized`, the tape is provably +//! garbage (every subsequent read goes through the `ArrayHeader`), so +//! `json_tape::release_tape_after_materialize` frees it right there without +//! waiting for any collector. That is the path `field_access` takes after +//! #7537's scan flip, which is why the fix does not depend on GC timing for +//! the workload that motivated it. + +use std::alloc::{alloc, dealloc, Layout}; +use std::cell::{Cell, RefCell}; + +use crate::json_tape::TapeEntry; + +/// Owned backing store for one tape. Frees on drop. +pub(crate) struct TapeSideAllocation { + ptr: *mut TapeEntry, + len: usize, +} + +impl TapeSideAllocation { + #[inline] + pub(crate) fn byte_len(&self) -> usize { + tape_layout(self.len).size() + } +} + +impl Drop for TapeSideAllocation { + fn drop(&mut self) { + if self.ptr.is_null() || self.len == 0 { + return; + } + unsafe { + dealloc(self.ptr as *mut u8, tape_layout(self.len)); + } + self.ptr = std::ptr::null_mut(); + self.len = 0; + } +} + +#[inline] +fn tape_layout(len: usize) -> Layout { + Layout::array::(len.max(1)).expect("tape length overflows a Layout") +} + +thread_local! { + /// `LazyArrayHeader` address -> its tape bytes. + static TAPE_REGISTRY: RefCell> = + RefCell::new(crate::fast_hash::new_ptr_hash_map()); + /// Live tape bytes on this thread — a local mirror of what this module + /// has handed to `gc_note_external_side_alloc`, so tests can cross-check + /// the registry against the counter without reading global GC state. + static TAPE_LIVE_BYTES: Cell = const { Cell::new(0) }; + /// Fast "this thread has never built a tape" gate, so the copying minor's + /// from-space pass and the sweep's dead-owner pass cost a single `Cell` + /// read on programs that never call `JSON.parse`. + static TAPE_REGISTRY_NONEMPTY: Cell = const { Cell::new(false) }; +} + +/// Allocate and fill a detached copy of `entries`. Uses the Rust global +/// allocator directly — this is deliberately NOT a GC allocation, so it never +/// enters arena/old-gen accounting and never runs a collection. +/// +/// Returns a null pointer for an empty tape; callers treat a null tape as +/// "length 0" and never dereference it. +pub(crate) fn allocate(entries: &[TapeEntry]) -> (*mut TapeEntry, TapeSideAllocation) { + if entries.is_empty() { + return ( + std::ptr::null_mut(), + TapeSideAllocation { + ptr: std::ptr::null_mut(), + len: 0, + }, + ); + } + let layout = tape_layout(entries.len()); + let raw = unsafe { alloc(layout) } as *mut TapeEntry; + assert!( + !raw.is_null(), + "json tape side allocation failed ({} bytes)", + layout.size() + ); + // GC_STORE_AUDIT(POINTER_FREE): TapeEntry is offset/kind/link numerics, no heap edges. + unsafe { + std::ptr::copy_nonoverlapping(entries.as_ptr(), raw, entries.len()); + } + let allocation = TapeSideAllocation { + ptr: raw, + len: entries.len(), + }; + note_allocated(allocation.byte_len()); + (raw, allocation) +} + +/// Hand ownership of `allocation` to `header_addr`. +/// +/// Must be called only once every allocation that could relocate the header is +/// behind us, because the key is the header's address. +pub(crate) fn register(header_addr: usize, allocation: TapeSideAllocation) { + if allocation.ptr.is_null() || allocation.len == 0 { + return; + } + TAPE_REGISTRY.with(|r| { + let mut registry = r.borrow_mut(); + assert!( + !registry.contains_key(&header_addr), + "lazy array tape registered twice for the same header" + ); + registry.insert(header_addr, allocation); + }); + TAPE_REGISTRY_NONEMPTY.with(|c| c.set(true)); +} + +/// Live tape bytes owned by this thread, as this module has accounted them. +#[cfg(test)] +#[inline] +fn live_bytes() -> usize { + TAPE_LIVE_BYTES.with(Cell::get) +} + +/// Drop the tape owned by `header_addr`, if any. Idempotent: the finalize +/// hook, the sweep-entry dead-owner pass, thread teardown, and the +/// deterministic post-materialize release all funnel through here, and any of +/// them may run first. +pub(crate) fn release(header_addr: usize) { + if !TAPE_REGISTRY_NONEMPTY.with(Cell::get) { + return; + } + let allocation = TAPE_REGISTRY.with(|r| { + let mut registry = r.borrow_mut(); + let taken = registry.remove(&header_addr); + if registry.is_empty() { + TAPE_REGISTRY_NONEMPTY.with(|c| c.set(false)); + } + taken + }); + let Some(allocation) = allocation else { + return; + }; + note_freed(allocation.byte_len()); + drop(allocation); +} + +/// Paired with [`note_freed`], and deliberately called from [`allocate`] — +/// BEFORE the owning header exists. +/// +/// `gc_note_external_side_alloc` can trigger a collection. At that point this +/// module holds nothing a collector can invalidate (the tape buffer is plain +/// `std::alloc` memory), and the caller holds only its rooted blob handle, so +/// there is no window where a live raw header could go stale across it. +#[inline] +fn note_allocated(bytes: usize) { + TAPE_LIVE_BYTES.with(|c| c.set(c.get().saturating_add(bytes))); + crate::gc::gc_note_external_side_alloc(bytes); +} + +/// Tape bytes are `external_side_live_bytes()` — the same old-generation +/// pressure term a `Map`'s entries buffer contributes, and for the same +/// reason: the owning header is old-gen, so only a FULL collection can prove a +/// retained tape dead, and those bytes have to be able to escalate that +/// reclaim or a `JSON.parse` loop that never materialises would grow without +/// bound. +/// +/// This does NOT reintroduce the pathology #7539 fixed. The old cost came from +/// tape bytes sitting in the old generation *as dead arena capacity* at +/// ~2.4 MB per parse; `field_access` now disowns each tape the moment +/// `materialized` is installed, so the term never accumulates there at all — +/// it holds one live tape at a time. `roundtrip`, which genuinely retains its +/// tape until the lazy array dies, keeps exactly the bounded cadence it has +/// today. +#[inline] +fn note_freed(bytes: usize) { + TAPE_LIVE_BYTES.with(|c| c.set(c.get().saturating_sub(bytes))); + crate::gc::gc_note_external_side_free(bytes); +} + +/// True when this thread has never registered a tape, so the collector's +/// per-cycle passes can skip the registry entirely. +#[inline] +pub(crate) fn registry_is_empty() -> bool { + !TAPE_REGISTRY_NONEMPTY.with(Cell::get) +} + +/// Registered owner addresses matching `is_dead`. Split from the release so +/// the caller can budget-chunk the frees the way the Map/Set sweep does. +pub(crate) fn collect_owners(is_dead: &dyn Fn(usize) -> bool) -> Vec { + if registry_is_empty() { + return Vec::new(); + } + TAPE_REGISTRY.with(|r| { + r.borrow() + .keys() + .copied() + .filter(|&addr| is_dead(addr)) + .collect() + }) +} + +/// Free every tape this thread still owns (thread teardown). +pub(crate) fn release_current_thread_lazy_tapes() { + let allocations = TAPE_REGISTRY.with(|r| { + r.borrow_mut() + .drain() + .map(|(_, allocation)| allocation) + .collect::>() + }); + TAPE_REGISTRY_NONEMPTY.with(|c| c.set(false)); + for allocation in allocations { + note_freed(allocation.byte_len()); + drop(allocation); + } +} + +/// Live tape bytes, cross-checked against the registry. +/// +/// At rest the running counter and the registry must agree — they are updated +/// by different code paths (`allocate`/`release` vs `register`/`remove`), and a +/// drift between them would mean either a tape freed while still owned or an +/// entry whose bytes were never accounted. +#[cfg(test)] +pub(crate) fn registered_bytes() -> usize { + let summed: usize = + TAPE_REGISTRY.with(|r| r.borrow().values().map(TapeSideAllocation::byte_len).sum()); + assert_eq!( + summed, + live_bytes(), + "tape byte counter drifted from the registry" + ); + summed +} diff --git a/crates/perry-runtime/src/json_tape_tests.rs b/crates/perry-runtime/src/json_tape_tests.rs index 1abc95c704..67bf149c90 100644 --- a/crates/perry-runtime/src/json_tape_tests.rs +++ b/crates/perry-runtime/src/json_tape_tests.rs @@ -121,6 +121,98 @@ fn tape_entry_layout() { ); } +/// #7539 requirement: the tape is POINTER-FREE BY CONSTRUCTION, which is what +/// licenses moving it out of the GC heap into a `json_tape_store` side +/// allocation that is never marked, scanned, or rewritten. +/// +/// The claim is structural, not a convention, and this pins the structure: +/// every `TapeEntry` field is an integer, and `offset`/`link` are `u32` — too +/// narrow to hold a 48-bit heap address even if some future code tried. `kind` +/// is a `u8`. There is exactly one writer of the region +/// (`json_tape_store::allocate`'s `copy_nonoverlapping` from a +/// `&[TapeEntry]`), so nothing can smuggle a reference in behind it. +/// +/// If someone widens a field to pointer size this fails, and the whole +/// direction has to be revisited: a tape that can carry a heap edge would need +/// tracing, and an untraced one would be a use-after-free. +#[test] +fn tape_entry_is_pointer_free_by_construction() { + // On every 64-bit target a struct with a pointer-sized field has + // alignment 8. `TapeEntry`'s alignment is 4, so no field it has — present + // or future — can hold a `*mut`/`usize`/`u64`. That is the whole proof, + // and it is checked by the compiler's own layout rules rather than by + // reading the field list and trusting it. + assert_eq!( + std::mem::align_of::(), + 4, + "TapeEntry gained a pointer-aligned field — it can no longer be \ + assumed pointer-free, and json_tape_store's untraced side \ + allocation would become a use-after-free" + ); + assert!( + std::mem::size_of::() <= 12, + "TapeEntry grew — recheck the pointer-free claim above" + ); + // Field widths, restated so a `u32 -> u64` widening fails here with a + // message that says why rather than only at the alignment assert. + let probe = TapeEntry { + offset: u32::MAX, + kind: u8::MAX, + link: u32::MAX, + }; + assert_eq!(std::mem::size_of_val(&probe.offset), 4); + assert_eq!(std::mem::size_of_val(&probe.kind), 1); + assert_eq!(std::mem::size_of_val(&probe.link), 4); +} + +/// The tape lives outside the header allocation now, so the header stays small +/// no matter how big the blob is. That is the property keeping it out of +/// `arena_alloc_gc`'s large-object arm — and therefore out of the old +/// generation, where only a FULL collection could reclaim it. +#[test] +fn lazy_array_header_stays_small_regardless_of_tape_size() { + assert!( + std::mem::size_of::() < crate::gc::LARGE_OBJECT_THRESHOLD_BYTES, + "LazyArrayHeader must stay under the large-object threshold" + ); + // Restated here because this file is where the header's shape is asserted: + // `.length` is an inlined raw u32 load at offset 0 (codegen contract), and + // #7537's scan-flip threshold must keep its shape. + assert_eq!(std::mem::offset_of!(LazyArrayHeader, cached_length), 0); + assert_eq!(scan_flip_threshold(10_000), 156); + assert_eq!(scan_flip_threshold(10), 64); +} + +/// A disowned tape reads as EMPTY rather than as freed memory. Every reader +/// checks `materialized` first, but the null guard is what makes a stale read +/// safe instead of a use-after-free, so pin it directly. +#[test] +fn disowned_tape_reads_as_empty() { + let input = b"[1,2,3]"; + let text = crate::string::js_string_from_bytes(input.as_ptr(), input.len() as u32); + let lazy = with_built_tape(input, |tape| unsafe { + alloc_lazy_array(tape, 0, count_array_length(tape, 0), text) + }) + .expect("valid JSON should build a tape"); + + assert!(!unsafe { LazyArrayHeader::tape_slice(lazy) }.is_empty()); + let arr = unsafe { force_materialize_lazy(lazy) }; + assert_eq!(unsafe { (*arr).length }, 3); + + unsafe { + assert!((*lazy).tape.is_null()); + assert!(LazyArrayHeader::tape_slice(lazy).is_empty()); + let scope = crate::gc::RuntimeHandleScope::new(); + let source = TapeSource::Lazy { + hdr_handle: scope.root_raw_mut_ptr(lazy), + }; + assert!( + source.entry(0).is_none(), + "a disowned tape must yield no entries" + ); + } +} + #[test] fn force_materialize_numeric_lazy_array_preserves_raw_payload() { let input = br#"[1,2.5,3]"#; diff --git a/crates/perry-runtime/src/lib.rs b/crates/perry-runtime/src/lib.rs index fc03817728..68d577a794 100644 --- a/crates/perry-runtime/src/lib.rs +++ b/crates/perry-runtime/src/lib.rs @@ -192,6 +192,7 @@ pub mod i18n; pub mod ios_game_loop; pub mod json; pub mod json_tape; +pub(crate) mod json_tape_store; pub mod jsx; /// HarmonyOS streaming media playback (`perry/media`) — drain-queue /// bridge to `@ohos.multimedia.media.AVPlayer`. Symbols mirror the per- diff --git a/scripts/raw_handle_debt_baseline.txt b/scripts/raw_handle_debt_baseline.txt index a6905f8ba4..806adbfbe9 100644 --- a/scripts/raw_handle_debt_baseline.txt +++ b/scripts/raw_handle_debt_baseline.txt @@ -1 +1 @@ -999 +998 diff --git a/scripts/raw_handle_debt_files.txt b/scripts/raw_handle_debt_files.txt index 7bcc88b64a..f06e39db7f 100644 --- a/scripts/raw_handle_debt_files.txt +++ b/scripts/raw_handle_debt_files.txt @@ -61,7 +61,7 @@ 26 crates/perry-runtime/src/json/reviver.rs 3 crates/perry-runtime/src/json/stringify.rs 1 crates/perry-runtime/src/json/stringify_scalars.rs -22 crates/perry-runtime/src/json_tape.rs +21 crates/perry-runtime/src/json_tape.rs 26 crates/perry-runtime/src/map.rs 2 crates/perry-runtime/src/module_require.rs 7 crates/perry-runtime/src/node_stream/async_iterator.rs