From e04dcdab3ae17da898aeca0748e8f8c7b6c4f92a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 6 Aug 2026 01:24:30 +0200 Subject: [PATCH 1/2] perf(runtime): cache the hot thread-locals so one alloc pays one _tlv_get_addr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Darwin every `thread_local!` access is an out-of-line `_tlv_get_addr` call, and two different thread-locals are two different descriptors — so N distinct thread-locals on one path cost N calls no matter how well it inlines. A single `{v, w}` object literal touched about a dozen of them, and `_tlv_get_addr` was 27.9% of self time on `gc-handoff/bench/churn.ts` with the collector idle. `tls_hot::HotTls` caches those addresses in one const-init thread-local. The storage does not move — every slot is the address of the existing `thread_local!` in its owning module, so init order, lazy init and destructor registration are unchanged. Slots are untyped so owning modules keep their storage private; `tls_hot::tests::cached_addresses_match_thread_locals` asserts every address/accessor pairing, which is what stands between a mis-wired `fill()` and a well-typed reference to the wrong object. Four things fell out of profiling the remainder: - `incremental_mark_barrier_value` read a thread-local on every heap-pointer store to prove a null pointer was still null. It now consults the existing process-global `PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT` first, the same authority generated code already trusts. `incremental_mark_barrier_enable` now arms that count *before* installing the thread-local pointer, so there is no window where the pointer is live and the count still reads idle — that ordering was merely tidy before and is load-bearing now. - `js_array_length` probed the Map and Set registries on every call. Monotone "has anything ever been registered" flags answer for programs that use neither, with no counter to get wrong. - The page-generation cache had **one** entry, while the write barrier classifies at least two unrelated addresses per store; they evicted each other and it missed on essentially every call. Now 4-way, behind an `UnsafeCell` — `Cell::get` copies the whole set, which cost more than the lookup it avoids. - `layout_forget_object` takes one `borrow_mut` per map instead of a `borrow` to test emptiness plus a second `borrow_mut` to remove. Measured on `gc-handoff/bench`, interleaved A/B, best-of-5 user CPU on a quiet host: churn 1.16x, cycles 1.10x, retain1 1.08x, retain 1.07x, deeplist 1.07x, tree 1.03x. Peak RSS flat (worst cell +0.9%). Program output byte-identical on all six. The collector is untouched: `gc_ratchet` measurements against clean `main` agree on all 108 compared metrics except `heap_used_bytes` on 2 of 12 probes (<=0.8%, and it moves on different probes per build — allocation-boundary jitter, not behaviour). Per-cycle `PERRY_GC_TRACE` on churn/tree/retain is unchanged, including tree copying volume at 0.017 GB / 0.2 M object-copies. Refs #7469. --- crates/perry-runtime/src/arena/allocators.rs | 73 ++--- crates/perry-runtime/src/arena/block.rs | 28 ++ crates/perry-runtime/src/arena/mod.rs | 5 + crates/perry-runtime/src/arena/page_meta.rs | 167 ++++++++--- crates/perry-runtime/src/gc/barrier.rs | 72 +++-- crates/perry-runtime/src/gc/hot_tls.rs | 140 +++++++++ crates/perry-runtime/src/gc/layout.rs | 122 ++++---- crates/perry-runtime/src/gc/mod.rs | 5 + crates/perry-runtime/src/gc/roots.rs | 2 + .../perry-runtime/src/gc/roots/temp_roots.rs | 46 ++- crates/perry-runtime/src/lib.rs | 2 + crates/perry-runtime/src/map.rs | 30 ++ crates/perry-runtime/src/set.rs | 19 ++ crates/perry-runtime/src/tls_hot.rs | 274 ++++++++++++++++++ 14 files changed, 826 insertions(+), 159 deletions(-) create mode 100644 crates/perry-runtime/src/gc/hot_tls.rs create mode 100644 crates/perry-runtime/src/tls_hot.rs diff --git a/crates/perry-runtime/src/arena/allocators.rs b/crates/perry-runtime/src/arena/allocators.rs index 6c228c6e45..e0273b0fa8 100644 --- a/crates/perry-runtime/src/arena/allocators.rs +++ b/crates/perry-runtime/src/arena/allocators.rs @@ -13,37 +13,38 @@ use super::*; /// per-class-instance hot path that uses the inline allocator. #[inline] pub fn arena_alloc(size: usize, align: usize) -> *mut u8 { - INLINE_STATE.with(|inline_s| unsafe { - let inline_ptr = inline_s.get(); - ARENA.with(|a| { - let arena_ptr = (*a).get(); - // Sync inline → block before allocating, if the inline - // state has been initialized. Borrows are deliberately - // short-lived: `arena_cell_alloc` runs the GC between two - // disjoint borrows, and the collector mutates BOTH the arena - // and `INLINE_STATE` (`Arena::resync_inline_to_current`). #7022. - if !(*inline_ptr).data.is_null() { - let offset = (*inline_ptr).offset; - let arena = &mut *arena_ptr; - let current = arena.current; - arena.blocks[current].offset = offset; - } - let ptr = crate::arena::arena_cell_alloc(arena_ptr, size, align); - // Resync block → inline (may have advanced to a new block). - if !(*inline_ptr).data.is_null() { - let (data, offset, block_size) = { - let arena = &*arena_ptr; - let block = &arena.blocks[arena.current]; - (block.data, block.offset, block.size) - }; - let inline = &mut *inline_ptr; - inline.data = data; - inline.offset = offset; - inline.size = block_size; - } - ptr - }) - }) + // #7469: both thread-locals come off the one cached hot-TLS base rather + // than two `_tlv_get_addr` calls. The comment above about "two extra TLS + // reads cost ~5-10ns" was measuring exactly that toll. + unsafe { + let inline_ptr = crate::arena::hot_inline_state(); + let arena_ptr = crate::arena::hot_arena(); + // Sync inline → block before allocating, if the inline + // state has been initialized. Borrows are deliberately + // short-lived: `arena_cell_alloc` runs the GC between two + // disjoint borrows, and the collector mutates BOTH the arena + // and `INLINE_STATE` (`Arena::resync_inline_to_current`). #7022. + if !(*inline_ptr).data.is_null() { + let offset = (*inline_ptr).offset; + let arena = &mut *arena_ptr; + let current = arena.current; + arena.blocks[current].offset = offset; + } + let ptr = crate::arena::arena_cell_alloc(arena_ptr, size, align); + // Resync block → inline (may have advanced to a new block). + if !(*inline_ptr).data.is_null() { + let (data, offset, block_size) = { + let arena = &*arena_ptr; + let block = &arena.blocks[arena.current]; + (block.data, block.offset, block.size) + }; + let inline = &mut *inline_ptr; + inline.data = data; + inline.offset = offset; + inline.size = block_size; + } + ptr + } } /// Allocate from the longlived arena (issue #179). Unlike `arena_alloc`, @@ -286,9 +287,9 @@ pub fn arena_alloc_gc(size: usize, align: usize, obj_type: u8) -> *mut u8 { // micro-benchmarks like object_create / binary_trees run their tight // loops. Walking an empty Vec was costing ~10ns per alloc (borrow, // iterate, drop) for nothing; this `Cell` check is ~1ns. - let reused = if crate::gc::ARENA_FREE_LIST_NONEMPTY.with(|c| c.get()) { - crate::gc::ARENA_FREE_LIST.with(|fl| { - let mut fl = fl.borrow_mut(); + let reused = if crate::gc::hot_arena_free_list_nonempty().get() { + { + let mut fl = crate::gc::hot_arena_free_list().borrow_mut(); // Find a slot that fits (exact or slightly larger) let mut best_idx = None; let mut best_waste = usize::MAX; @@ -304,13 +305,13 @@ pub fn arena_alloc_gc(size: usize, align: usize, obj_type: u8) -> *mut u8 { if let Some(idx) = best_idx { let (ptr, _slot_size) = fl.swap_remove(idx); if fl.is_empty() { - crate::gc::ARENA_FREE_LIST_NONEMPTY.with(|c| c.set(false)); + crate::gc::hot_arena_free_list_nonempty().set(false); } Some(ptr) } else { None } - }) + } } else { None }; diff --git a/crates/perry-runtime/src/arena/block.rs b/crates/perry-runtime/src/arena/block.rs index fcfc151cf1..59ca0b6766 100644 --- a/crates/perry-runtime/src/arena/block.rs +++ b/crates/perry-runtime/src/arena/block.rs @@ -858,6 +858,34 @@ thread_local! { }) }; } +// --- #7469 hot-TLS address providers. See `crate::tls_hot`. --- + +/// Address of this thread's `ARENA`. Resolving it once and caching it is what +/// lets the allocation path stop paying `_tlv_get_addr` per access. +pub(crate) fn arena_hot_addr() -> *mut u8 { + ARENA.with(|a| a.get() as *mut u8) +} + +/// Address of this thread's `INLINE_STATE`. `js_inline_arena_state` already +/// hands this same pointer to generated code, so caching it here adds no new +/// exposure. +pub(crate) fn inline_state_hot_addr() -> *mut u8 { + INLINE_STATE.with(|s| s.get() as *mut u8) +} + +/// This thread's nursery arena, one cached load instead of a TLS resolution. +#[inline(always)] +pub(crate) fn hot_arena() -> *mut Arena { + crate::tls_hot::hot().arena as *mut Arena +} + +/// This thread's inline bump-allocator state, one cached load instead of a TLS +/// resolution. +#[inline(always)] +pub(crate) fn hot_inline_state() -> *mut InlineArenaState { + crate::tls_hot::hot().inline_state as *mut InlineArenaState +} + /// Delta-maintenance for `OLD_GEN_IN_USE_BYTES` — see the thread-local's /// doc comment for the full mutation-site inventory. #[inline] diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index 5dbaff7c99..0d3cad7606 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -33,6 +33,10 @@ pub(crate) use block::{ ARENA, ARENA_TOTAL_BYTES, BLOCK_SIZE, FRESH_GENERAL_BLOCK_MIN_USED_BYTES, INLINE_STATE, LONGLIVED_ARENA, OLD_ARENA, OLD_GEN_IN_USE_BYTES, SURVIVOR_ARENA_0, SURVIVOR_ARENA_1, }; +/// #7469 hot-TLS plumbing — see `crate::tls_hot`. The `*_hot_addr` half is +/// consumed by `tls_hot::fill`; the `hot_*` half is the cached accessor the +/// allocation path uses instead of a per-access `_tlv_get_addr`. +pub(crate) use block::{arena_hot_addr, hot_arena, hot_inline_state, inline_state_hot_addr}; #[cfg(test)] pub(crate) use block::{ block_pool_bytes_for_test, force_next_block_alloc_failure, gc_trigger_arena_borrow_depth, @@ -43,6 +47,7 @@ pub(crate) use page_meta::{ unregister_block_generation, unregister_old_block_pages, OLD_GEN_RECLAIM_RETURNED_BYTES, OLD_GEN_RECLAIM_REUSABLE_BYTES, }; +pub(crate) use page_meta::{page_generation_cache_hot_addr, page_generations_hot_addr}; // --- Public API (explicit named re-exports) --- diff --git a/crates/perry-runtime/src/arena/page_meta.rs b/crates/perry-runtime/src/arena/page_meta.rs index 0368bced4c..0b4653b17e 100644 --- a/crates/perry-runtime/src/arena/page_meta.rs +++ b/crates/perry-runtime/src/arena/page_meta.rs @@ -108,6 +108,69 @@ impl PageGenerationCache { } } +/// Ways in the [`PageGenerationCacheSet`] below. +/// +/// #7469: this was a **one**-entry cache, and the write barrier classifies at +/// least two unrelated addresses per store — the child being written and the +/// object being written into. On `churn.ts` those live in different 1 MiB +/// generation classes, so consecutive classifications evicted each other and +/// the "cache" missed on essentially every call: 71 self samples in the +/// authoritative map lookup that the cache exists to avoid. Four ways covers +/// the barrier's working set (child, parent, array payload, header) with room +/// to spare; it is still a fixed-size array probed linearly, so a hit is a few +/// compares off one cache line. +const PAGE_GENERATION_CACHE_WAYS: usize = 4; + +/// Small direct-probed cache in front of [`PageGenerationMap`]. +/// +/// Pure accelerator: a miss, a stale way, or a full set all fall through to +/// the authoritative map, so the only thing correctness depends on is that +/// every invalidation clears **all** ways — which is why +/// [`invalidate_generation_cache`] resets the whole set rather than one entry. +/// +/// Stored behind an `UnsafeCell`, not a `Cell`: `Cell::get` returns a **copy**, +/// and copying ~200 bytes on every classification cost more than the map lookup +/// the cache exists to avoid (measured as a ~2% regression on `retain.ts` +/// before this was switched). Access is single-threaded by construction — the +/// cache is thread-local and no path holds a reference across a call that could +/// re-enter classification. +#[derive(Clone, Copy)] +struct PageGenerationCacheSet { + ways: [PageGenerationCache; PAGE_GENERATION_CACHE_WAYS], + /// Round-robin victim for the next insert. + next: usize, +} + +impl PageGenerationCacheSet { + const fn empty() -> Self { + Self { + ways: [PageGenerationCache::empty(); PAGE_GENERATION_CACHE_WAYS], + next: 0, + } + } + + #[inline(always)] + fn lookup(&self, key: usize, addr: usize) -> Option { + for way in self.ways.iter() { + if way.valid && way.key == key && way.range.contains(addr) { + return Some(way.range); + } + } + None + } + + #[inline] + fn insert(&mut self, key: usize, range: PageGenerationRange) { + let slot = self.next % PAGE_GENERATION_CACHE_WAYS; + self.ways[slot] = PageGenerationCache { + key, + range, + valid: true, + }; + self.next = slot.wrapping_add(1); + } +} + /// #7187: this map used to carry a bespoke identity hasher (`write_usize` /// stored the key verbatim). `HashMap` is hashbrown, which takes the bucket /// index from the hash's LOW bits and the SIMD control byte from @@ -242,8 +305,8 @@ thread_local! { static PAGE_GENERATIONS: RefCell = RefCell::new(crate::fast_hash::new_ptr_hash_map()); - static PAGE_GENERATION_CACHE: Cell = - const { Cell::new(PageGenerationCache::empty()) }; + static PAGE_GENERATION_CACHE: UnsafeCell = + const { UnsafeCell::new(PageGenerationCacheSet::empty()) }; static OLD_GEN_PAGE_OBJECTS: RefCell = RefCell::new(crate::fast_hash::new_ptr_hash_map()); @@ -262,6 +325,34 @@ thread_local! { static OLD_GEN_PAGE_DIRTY_EPOCH: Cell = const { Cell::new(1) }; } +// --- #7469 hot-TLS address providers. See `crate::tls_hot`. --- + +/// Address of this thread's `PAGE_GENERATION_CACHE`. +pub(crate) fn page_generation_cache_hot_addr() -> *mut u8 { + PAGE_GENERATION_CACHE.with(|c| c.get() as *mut u8) +} + +/// Address of this thread's `PAGE_GENERATIONS`. +pub(crate) fn page_generations_hot_addr() -> *mut u8 { + PAGE_GENERATIONS.with(|p| p as *const _ as *mut u8) +} + +/// [`PAGE_GENERATION_CACHE`] without a TLS resolution — see `crate::tls_hot`. +#[inline(always)] +fn hot_page_generation_cache() -> *mut PageGenerationCacheSet { + // SAFETY: the slot is filled from `page_generation_cache_hot_addr` above, + // and `tls_hot::tests::cached_addresses_match_thread_locals` asserts the + // pairing. + crate::tls_hot::hot().page_generation_cache as *mut PageGenerationCacheSet +} + +/// [`PAGE_GENERATIONS`] without a TLS resolution — see `crate::tls_hot`. +#[inline(always)] +fn hot_page_generations() -> &'static RefCell { + // SAFETY: as above, paired with `page_generations_hot_addr`. + unsafe { &*(crate::tls_hot::hot().page_generations as *const RefCell) } +} + #[inline] fn old_gen_page_dirty_epoch() -> u64 { OLD_GEN_PAGE_DIRTY_EPOCH.with(|epoch| epoch.get()) @@ -284,7 +375,9 @@ pub(crate) fn generation_page_base(page: usize) -> usize { #[inline] fn invalidate_generation_cache() { - PAGE_GENERATION_CACHE.with(|cache| cache.set(PageGenerationCache::empty())); + // Every way, not one — a stale way is exactly what this guards against. + // SAFETY: thread-local, single-threaded. + PAGE_GENERATION_CACHE.with(|cache| unsafe { *cache.get() = PageGenerationCacheSet::empty() }); } fn register_old_block_pages(base: usize, size: usize) { @@ -430,32 +523,42 @@ pub(crate) fn unregister_block_generation(base: usize, size: usize) { invalidate_generation_cache(); } -#[inline] +/// #7469: split so the **cache-hit** arm is small enough to actually inline +/// into its callers. A single `js_write_barrier_slot` classifies twice (child +/// then parent) and `write_barrier_decoded_parent` classifies again; with the +/// miss path inlined alongside, the whole thing stayed out of line and each +/// call paid its own `_tlv_get_addr`. Out-of-lining the miss lets the hit arm +/// inline, and LLVM then CSEs the one remaining hot-TLS resolution across +/// every classification in the barrier. +#[inline(always)] pub(crate) fn classify_heap_generation(addr: usize) -> HeapGeneration { if addr == 0 { return HeapGeneration::Unknown; } let key = generation_class_key_for_addr(addr); - if let Some(generation) = PAGE_GENERATION_CACHE.with(|cache| { - let cached = cache.get(); - (cached.valid && cached.key == key && cached.range.contains(addr)) - .then_some(cached.range.generation) - }) { - return generation; + // Both tables come off the one cached hot-TLS base — this runs on every + // `decode_heap_addr`, i.e. every heap store the write barrier sees, and was + // the single largest `_tlv_get_addr` caller on `churn.ts` (116 of 653 + // attributed samples) precisely because it resolved two distinct + // thread-locals per call. + // SAFETY: thread-local, single-threaded, and the borrow ends here. + if let Some(range) = unsafe { (*hot_page_generation_cache()).lookup(key, addr) } { + return range.generation; } + classify_heap_generation_uncached(addr, key) +} - let found = PAGE_GENERATIONS.with(|pages| { - let pages = pages.borrow(); +/// Cache-miss arm of [`classify_heap_generation`]: consult the page map and +/// re-prime the one-entry cache. +#[inline(never)] +fn classify_heap_generation_uncached(addr: usize, key: usize) -> HeapGeneration { + let found = { + let pages = hot_page_generations().borrow(); pages.get(&key).and_then(|slot| slot.find(addr)) - }); + }; if let Some(range) = found { - PAGE_GENERATION_CACHE.with(|cache| { - cache.set(PageGenerationCache { - key, - range, - valid: true, - }); - }); + // SAFETY: as above. + unsafe { (*hot_page_generation_cache()).insert(key, range) }; range.generation } else { HeapGeneration::Unknown @@ -468,26 +571,18 @@ pub(crate) fn classify_heap_space(addr: usize) -> HeapSpace { return HeapSpace::Unknown; } let key = generation_class_key_for_addr(addr); - if let Some(space) = PAGE_GENERATION_CACHE.with(|cache| { - let cached = cache.get(); - (cached.valid && cached.key == key && cached.range.contains(addr)) - .then_some(cached.range.space) - }) { - return space; + // SAFETY: thread-local, single-threaded, and the borrow ends here. + if let Some(range) = unsafe { (*hot_page_generation_cache()).lookup(key, addr) } { + return range.space; } - let found = PAGE_GENERATIONS.with(|pages| { - let pages = pages.borrow(); + let found = { + let pages = hot_page_generations().borrow(); pages.get(&key).and_then(|slot| slot.find(addr)) - }); + }; if let Some(range) = found { - PAGE_GENERATION_CACHE.with(|cache| { - cache.set(PageGenerationCache { - key, - range, - valid: true, - }); - }); + // SAFETY: as above. + unsafe { (*hot_page_generation_cache()).insert(key, range) }; range.space } else { HeapSpace::Unknown diff --git a/crates/perry-runtime/src/gc/barrier.rs b/crates/perry-runtime/src/gc/barrier.rs index 4edfcfd808..d67de9fc2d 100644 --- a/crates/perry-runtime/src/gc/barrier.rs +++ b/crates/perry-runtime/src/gc/barrier.rs @@ -1,3 +1,6 @@ +use super::hot_tls::{ + hot_birth_extra_flags, hot_incremental_mark_minor_only, hot_incremental_mark_valid_ptrs, +}; use super::*; /// Snapshot the remembered dirty ranges before the collection clears them. @@ -672,14 +675,21 @@ pub static PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT: std::sync::atomic::Atomi pub(super) fn incremental_mark_barrier_enable(valid_ptrs: &ValidPointerSet, minor_only: bool) { INCREMENTAL_MARK_BARRIER_MINOR_ONLY.with(|cell| cell.set(minor_only)); - let newly_active = INCREMENTAL_MARK_BARRIER_VALID_PTRS.with(|cell| { - let newly_active = cell.get().is_null(); - cell.set(valid_ptrs as *const ValidPointerSet); - newly_active - }); + // Arm the global count BEFORE installing the thread-local pointer, and + // (in `disable`) decrement it AFTER clearing the pointer. Both halves keep + // the count conservatively armed across the whole window in which the + // pointer is live. + // + // #7469 made this ordering load-bearing rather than merely tidy: + // `incremental_mark_barrier_value` now treats a zero count as proof that + // no shading is needed, so a window where the pointer is installed but the + // count is not yet bumped would be a window where a store silently skips + // its insertion barrier — a lost mark, i.e. a live object swept. + let newly_active = INCREMENTAL_MARK_BARRIER_VALID_PTRS.with(|cell| cell.get().is_null()); if newly_active { PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT.fetch_add(1, Ordering::SeqCst); } + INCREMENTAL_MARK_BARRIER_VALID_PTRS.with(|cell| cell.set(valid_ptrs as *const ValidPointerSet)); } pub(super) fn incremental_mark_barrier_disable() { @@ -705,11 +715,30 @@ pub(super) fn incremental_mark_barrier_disable() { GC_BIRTH_EXTRA_FLAGS.with(|cell| cell.set(0)); } +/// True when no thread anywhere has an incremental mark barrier installed. +/// +/// The same authority `PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT` already +/// gives generated code (see its doc comment): zero proves *this* thread's +/// `INCREMENTAL_MARK_BARRIER_VALID_PTRS` is null, because a thread arming its +/// own barrier increments the count before any store can observe the armed +/// pointer. Non-zero is conservative and falls through to the thread-local +/// read, which then finds its own null and returns. +/// +/// #7469: the point is to skip the *thread-local* read. On Darwin that read is +/// an out-of-line `_tlv_get_addr` call on every heap-pointer store, and it was +/// 91 of the 653 attributed `_tlv_get_addr` samples on `churn.ts` — all of them +/// spent proving a null pointer was still null. This is a relaxed load of a +/// static: `adrp` + `ldr` and a perfectly-predicted branch. +#[inline(always)] +fn incremental_mark_barrier_globally_idle() -> bool { + PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT.load(Ordering::Relaxed) == 0 +} + /// Allocate-black birth flags for runtime-path allocations — see /// `GC_BIRTH_EXTRA_FLAGS`. #[inline(always)] pub fn gc_birth_extra_flags() -> u8 { - GC_BIRTH_EXTRA_FLAGS.with(|cell| cell.get()) + hot_birth_extra_flags().get() } /// A born-black object must also be TRACED: marking treats MARKED as @@ -725,7 +754,7 @@ pub fn gc_birth_extra_flags() -> u8 { /// visit per mid-cycle runtime allocation. #[inline] pub(crate) fn gc_note_black_birth(header: *mut GcHeader) { - if GC_BIRTH_EXTRA_FLAGS.with(|cell| cell.get()) & GC_FLAG_MARKED == 0 { + if hot_birth_extra_flags().get() & GC_FLAG_MARKED == 0 { return; } // Leaf types (strings, pointer-free payloads) carry no child edges — the @@ -740,7 +769,10 @@ pub(crate) fn gc_note_black_birth(header: *mut GcHeader) { #[inline] pub(super) fn incremental_mark_barrier_active() -> bool { - INCREMENTAL_MARK_BARRIER_VALID_PTRS.with(|cell| !cell.get().is_null()) + if incremental_mark_barrier_globally_idle() { + return false; + } + !hot_incremental_mark_valid_ptrs().get().is_null() } #[inline] @@ -820,9 +852,7 @@ fn incremental_mark_barrier_value_with_valid_ptrs( // Minor cycles shade only nursery children (see the // INCREMENTAL_MARK_BARRIER_MINOR_ONLY doc: stray old-gen marks survive a // minor's sweep and poison the next full cycle's trace). - if INCREMENTAL_MARK_BARRIER_MINOR_ONLY.with(|cell| cell.get()) - && !crate::arena::pointer_in_nursery(addr) - { + if hot_incremental_mark_minor_only().get() && !crate::arena::pointer_in_nursery(addr) { return false; } unsafe { @@ -837,14 +867,18 @@ fn incremental_mark_barrier_value_with_valid_ptrs( } pub(super) fn incremental_mark_barrier_value(value_bits: u64) -> bool { - INCREMENTAL_MARK_BARRIER_VALID_PTRS.with(|cell| { - let ptr = cell.get(); - if ptr.is_null() { - return false; - } - let valid_ptrs = unsafe { &*ptr }; - incremental_mark_barrier_value_with_valid_ptrs(value_bits, valid_ptrs) - }) + // #7469: the overwhelmingly common case is "no cycle anywhere", and + // proving it must not cost a thread-local resolution — this runs on every + // heap-pointer store in compiled code. + if incremental_mark_barrier_globally_idle() { + return false; + } + let ptr = hot_incremental_mark_valid_ptrs().get(); + if ptr.is_null() { + return false; + } + let valid_ptrs = unsafe { &*ptr }; + incremental_mark_barrier_value_with_valid_ptrs(value_bits, valid_ptrs) } #[allow(dead_code)] diff --git a/crates/perry-runtime/src/gc/hot_tls.rs b/crates/perry-runtime/src/gc/hot_tls.rs new file mode 100644 index 0000000000..6a7aa32920 --- /dev/null +++ b/crates/perry-runtime/src/gc/hot_tls.rs @@ -0,0 +1,140 @@ +//! The `gc` half of the #7469 hot-thread-local plumbing. +//! +//! Two kinds of function live here, one pair per thread-local: +//! +//! - `…_hot_addr()` — resolves the thread-local's address the ordinary way. +//! Called once per thread by [`crate::tls_hot::fill`], which lives outside +//! `gc` and so needs these re-exported at `pub(crate)`. +//! - `hot_…()` — reads the cached address back and casts it to the owning +//! type. This is what the hot paths call instead of `KEY.with(…)`, and it is +//! why a single `{v, w}` object literal no longer pays a `_tlv_get_addr` +//! call per side table it touches. +//! +//! The casts are the reason the pairs sit together in one file: the cache +//! stores untyped `*mut u8` (so each owning module keeps its storage type +//! private), so a `hot_…()` paired with the wrong `…_hot_addr()` would hand +//! out a well-typed reference to the wrong object. +//! `tls_hot::tests::cached_addresses_match_thread_locals` asserts every pairing +//! and is the guard against exactly that. +//! +//! Split out of `barrier.rs` and `layout.rs` to stay under the repo's +//! 2000-line-per-file cap (`scripts/check_file_size.sh`). + +use super::barrier::{ + GC_BIRTH_EXTRA_FLAGS, INCREMENTAL_MARK_BARRIER_MINOR_ONLY, INCREMENTAL_MARK_BARRIER_VALID_PTRS, +}; +use super::layout::{ + LayoutSlotMask, TypedLayoutDescriptor, LAYOUT_SLOT_MASKS, SHAPE_LAYOUTS, TYPED_LAYOUTS, +}; +use super::malloc::{ARENA_FREE_LIST, ARENA_FREE_LIST_NONEMPTY}; +use super::trace::ValidPointerSet; +use std::cell::{Cell, RefCell}; + +// --- gc::barrier ------------------------------------------------------------ + +/// Address of this thread's `GC_BIRTH_EXTRA_FLAGS`. +pub(crate) fn birth_extra_flags_hot_addr() -> *mut u8 { + GC_BIRTH_EXTRA_FLAGS.with(|c| c as *const _ as *mut u8) +} + +/// Address of this thread's `INCREMENTAL_MARK_BARRIER_VALID_PTRS`. +pub(crate) fn incremental_mark_valid_ptrs_hot_addr() -> *mut u8 { + INCREMENTAL_MARK_BARRIER_VALID_PTRS.with(|c| c as *const _ as *mut u8) +} + +/// Address of this thread's `INCREMENTAL_MARK_BARRIER_MINOR_ONLY`. +pub(crate) fn incremental_mark_minor_only_hot_addr() -> *mut u8 { + INCREMENTAL_MARK_BARRIER_MINOR_ONLY.with(|c| c as *const _ as *mut u8) +} + +/// `GC_BIRTH_EXTRA_FLAGS` without a TLS resolution. +#[inline(always)] +pub(super) fn hot_birth_extra_flags() -> &'static Cell { + // SAFETY: paired with `birth_extra_flags_hot_addr` above. + unsafe { &*(crate::tls_hot::hot().birth_extra_flags as *const Cell) } +} + +/// `INCREMENTAL_MARK_BARRIER_VALID_PTRS` without a TLS resolution. +#[inline(always)] +pub(super) fn hot_incremental_mark_valid_ptrs() -> &'static Cell<*const ValidPointerSet> { + // SAFETY: paired with `incremental_mark_valid_ptrs_hot_addr` above. + unsafe { + &*(crate::tls_hot::hot().incremental_mark_valid_ptrs as *const Cell<*const ValidPointerSet>) + } +} + +/// `INCREMENTAL_MARK_BARRIER_MINOR_ONLY` without a TLS resolution. +#[inline(always)] +pub(super) fn hot_incremental_mark_minor_only() -> &'static Cell { + // SAFETY: paired with `incremental_mark_minor_only_hot_addr` above. + unsafe { &*(crate::tls_hot::hot().incremental_mark_minor_only as *const Cell) } +} + +// --- gc::layout ------------------------------------------------------------- + +type SlotMaskMap = crate::fast_hash::PtrHashMap; +type TypedLayoutMap = crate::fast_hash::PtrHashMap; +type ShapeLayoutMap = crate::fast_hash::PtrHashMap>; + +/// Address of this thread's `LAYOUT_SLOT_MASKS`. +pub(crate) fn layout_slot_masks_hot_addr() -> *mut u8 { + LAYOUT_SLOT_MASKS.with(|m| m as *const _ as *mut u8) +} + +/// Address of this thread's `TYPED_LAYOUTS`. +pub(crate) fn typed_layouts_hot_addr() -> *mut u8 { + TYPED_LAYOUTS.with(|m| m as *const _ as *mut u8) +} + +/// Address of this thread's `SHAPE_LAYOUTS`. +pub(crate) fn shape_layouts_hot_addr() -> *mut u8 { + SHAPE_LAYOUTS.with(|m| m as *const _ as *mut u8) +} + +/// `LAYOUT_SLOT_MASKS` without a TLS resolution. +#[inline(always)] +pub(super) fn hot_layout_slot_masks() -> &'static RefCell { + // SAFETY: paired with `layout_slot_masks_hot_addr` above. + unsafe { &*(crate::tls_hot::hot().layout_slot_masks as *const RefCell) } +} + +/// `TYPED_LAYOUTS` without a TLS resolution. +#[inline(always)] +pub(super) fn hot_typed_layouts() -> &'static RefCell { + // SAFETY: paired with `typed_layouts_hot_addr` above. + unsafe { &*(crate::tls_hot::hot().typed_layouts as *const RefCell) } +} + +/// `SHAPE_LAYOUTS` without a TLS resolution. +#[inline(always)] +pub(super) fn hot_shape_layouts() -> &'static RefCell { + // SAFETY: paired with `shape_layouts_hot_addr` above. + unsafe { &*(crate::tls_hot::hot().shape_layouts as *const RefCell) } +} + +// --- gc::malloc ------------------------------------------------------------- + +/// Address of this thread's `ARENA_FREE_LIST`. +pub(crate) fn arena_free_list_hot_addr() -> *mut u8 { + ARENA_FREE_LIST.with(|f| f as *const _ as *mut u8) +} + +/// Address of this thread's `ARENA_FREE_LIST_NONEMPTY`. +pub(crate) fn arena_free_list_nonempty_hot_addr() -> *mut u8 { + ARENA_FREE_LIST_NONEMPTY.with(|f| f as *const _ as *mut u8) +} + +/// `ARENA_FREE_LIST` without a TLS resolution. +#[inline(always)] +pub(crate) fn hot_arena_free_list() -> &'static RefCell> { + // SAFETY: paired with `arena_free_list_hot_addr` above. + unsafe { &*(crate::tls_hot::hot().arena_free_list as *const RefCell>) } +} + +/// `ARENA_FREE_LIST_NONEMPTY` without a TLS resolution — the "is there anything +/// to reuse?" probe every `arena_alloc_gc` pays. +#[inline(always)] +pub(crate) fn hot_arena_free_list_nonempty() -> &'static Cell { + // SAFETY: paired with `arena_free_list_nonempty_hot_addr` above. + unsafe { &*(crate::tls_hot::hot().arena_free_list_nonempty as *const Cell) } +} diff --git a/crates/perry-runtime/src/gc/layout.rs b/crates/perry-runtime/src/gc/layout.rs index 1e88fca8fe..e3f9132bab 100644 --- a/crates/perry-runtime/src/gc/layout.rs +++ b/crates/perry-runtime/src/gc/layout.rs @@ -1,3 +1,4 @@ +use super::hot_tls::{hot_layout_slot_masks, hot_shape_layouts, hot_typed_layouts}; use super::*; // Copied-nursery survival age stored in otherwise-unused low @@ -292,9 +293,9 @@ pub(super) struct TypedLayoutDescriptor { // NaN-boxing tag constants (duplicated from value.rs to avoid circular deps) thread_local! { - pub(super) static LAYOUT_SLOT_MASKS: RefCell> = + pub(in crate::gc) static LAYOUT_SLOT_MASKS: RefCell> = RefCell::new(crate::fast_hash::new_ptr_hash_map()); - pub(super) static TYPED_LAYOUTS: RefCell> = + pub(in crate::gc) static TYPED_LAYOUTS: RefCell> = RefCell::new(crate::fast_hash::new_ptr_hash_map()); #[cfg(test)] pub(super) static TRACE_SLOT_READS: Cell = const { Cell::new(0) }; @@ -318,10 +319,35 @@ thread_local! { // conservative scan — never a wrong descriptor (mirrors the ShapeTable trust // model). Nothing to prune on object death (entries are per-shape, shared). thread_local! { - static SHAPE_LAYOUTS: RefCell>> = + pub(in crate::gc) static SHAPE_LAYOUTS: RefCell>> = RefCell::new(crate::fast_hash::new_ptr_hash_map()); } +/// Drop any per-object layout record keyed by `user_ptr`. +/// +/// Both maps are probed on **every** object allocation +/// ([`layout_init_pointer_free`]) and again on every typed-shape install, to +/// clear whatever a previous tenant of a recycled address left behind. The +/// `is_empty()` guards skip the hash of an address that provably cannot be +/// present: with #6893 shape-keying, `TYPED_LAYOUTS` holds only objects that +/// diverged from their shape, so on a well-behaved workload it is empty and +/// the probe is pure cost. +#[inline] +fn layout_forget_object(user_ptr: usize) { + // One `borrow_mut` per map, not a `borrow` to test emptiness followed by a + // second `borrow_mut` to remove: `RefCell`'s flag traffic is a measurable + // share of a function this hot (#7469). + let mut masks = hot_layout_slot_masks().borrow_mut(); + if !masks.is_empty() { + masks.remove(&user_ptr); + } + drop(masks); + let mut typed = hot_typed_layouts().borrow_mut(); + if !typed.is_empty() { + typed.remove(&user_ptr); + } +} + fn shape_layout_keyed_enabled() -> bool { use std::sync::OnceLock; static E: OnceLock = OnceLock::new(); @@ -375,14 +401,12 @@ unsafe fn with_shape_shared_descriptor( // a shape with a different field count (moving-GC relocation before the new // address is re-installed). Fall back (per-object → conservative). let field_count = (*(user_ptr as *const crate::object::ObjectHeader)).field_count as usize; - SHAPE_LAYOUTS.with(|m| { - let map = m.borrow(); - let desc = map.get(&keys)?.as_ref()?; - if desc.slot_count != field_count { - return None; - } - Some(f(desc)) - }) + let map = hot_shape_layouts().borrow(); + let desc = map.get(&keys)?.as_ref()?; + if desc.slot_count != field_count { + return None; + } + Some(f(desc)) } /// Cloning form of [`with_shape_shared_descriptor`], for the callers that need @@ -457,28 +481,24 @@ unsafe fn shape_install_shared( header: *mut GcHeader, descriptor: &TypedLayoutDescriptor, ) -> bool { - let mut shared_ok = false; - SHAPE_LAYOUTS.with(|m| { - let mut m = m.borrow_mut(); + let shared_ok = { + let mut m = hot_shape_layouts().borrow_mut(); match m.get(&keys) { None => { m.insert(keys, Some(descriptor.clone())); - shared_ok = true; - } - Some(Some(existing)) if existing == descriptor => { - shared_ok = true; + true } + Some(Some(existing)) if existing == descriptor => true, Some(Some(_)) => { // Same keys, different layout ⟹ ambiguous. Poison the entry so // future lookups (and any still-INTACT siblings) fall back. m.insert(keys, None); - shared_ok = false; - } - Some(None) => { - shared_ok = false; // already ambiguous + false } + // Already ambiguous. + Some(None) => false, } - }); + }; if shared_ok { header_set_typed_layout_intact(header); if descriptor.pointer_mask.is_empty() { @@ -571,12 +591,7 @@ pub(crate) unsafe fn layout_init_pointer_free(user_ptr: *mut u8) { return; }; set_layout_state(header, GC_LAYOUT_POINTER_FREE); - LAYOUT_SLOT_MASKS.with(|m| { - m.borrow_mut().remove(&(user_ptr as usize)); - }); - TYPED_LAYOUTS.with(|m| { - m.borrow_mut().remove(&(user_ptr as usize)); - }); + layout_forget_object(user_ptr as usize); header_clear_typed_layout_intact(header); } @@ -590,12 +605,7 @@ pub(crate) unsafe fn layout_init_all_pointer_slots(user_ptr: *mut u8) { return; }; header_clear_typed_layout_intact(header); - TYPED_LAYOUTS.with(|m| { - m.borrow_mut().remove(&(user_ptr as usize)); - }); - LAYOUT_SLOT_MASKS.with(|m| { - m.borrow_mut().remove(&(user_ptr as usize)); - }); + layout_forget_object(user_ptr as usize); set_layout_state(header, GC_LAYOUT_SIDE_MASK); (*header)._reserved |= GC_LAYOUT_ALL_POINTERS; } @@ -706,8 +716,10 @@ pub(crate) fn layout_note_slot(parent_user: usize, slot_index: usize, value_bits // #6893: per-object descriptor (diverged/ambiguous objects) OR the // shared shape descriptor (the common INTACT case). Exactly one is // present for an INTACT object. - let typed = TYPED_LAYOUTS - .with(|m| m.borrow().get(&parent_user).cloned()) + let typed = hot_typed_layouts() + .borrow() + .get(&parent_user) + .cloned() .or_else(|| shape_shared_descriptor(parent_user)); if let Some(typed) = typed { if slot_index >= typed.slot_count { @@ -743,8 +755,8 @@ pub(crate) fn layout_note_slot(parent_user: usize, slot_index: usize, value_bits if !pointer && (*header)._reserved & GC_LAYOUT_STATE_MASK == GC_LAYOUT_POINTER_FREE { return; } - LAYOUT_SLOT_MASKS.with(|m| { - let mut masks = m.borrow_mut(); + { + let mut masks = hot_layout_slot_masks().borrow_mut(); if pointer { if let Some(mask) = masks.get_mut(&parent_user) { mask.set_slot(slot_index); @@ -777,7 +789,7 @@ pub(crate) fn layout_note_slot(parent_user: usize, slot_index: usize, value_bits set_layout_state(header, GC_LAYOUT_POINTER_FREE); } } - }); + } } } @@ -921,28 +933,28 @@ unsafe fn init_typed_shape_layout( 0 }; if keys != 0 && shape_install_shared(keys, header, &descriptor) { - TYPED_LAYOUTS.with(|m| { - m.borrow_mut().remove(&user_ptr); - }); - LAYOUT_SLOT_MASKS.with(|m| { - m.borrow_mut().remove(&user_ptr); - }); + // The common path for every object literal: the shape already owns a + // canonical descriptor, so this object needs no per-object record at + // all. `layout_forget_object` skips the hash entirely when the maps + // are empty, which on a monomorphic workload they are. + layout_forget_object(user_ptr); return; } - TYPED_LAYOUTS.with(|m| { - m.borrow_mut().insert(user_ptr, descriptor); - }); + hot_typed_layouts() + .borrow_mut() + .insert(user_ptr, descriptor); header_set_typed_layout_intact(header); if pointer_mask.is_empty() { set_layout_state(header, GC_LAYOUT_POINTER_FREE); - LAYOUT_SLOT_MASKS.with(|m| { - m.borrow_mut().remove(&user_ptr); - }); + let masks = hot_layout_slot_masks(); + if !masks.borrow().is_empty() { + masks.borrow_mut().remove(&user_ptr); + } } else { set_layout_state(header, GC_LAYOUT_SIDE_MASK); - LAYOUT_SLOT_MASKS.with(|m| { - m.borrow_mut().insert(user_ptr, pointer_mask); - }); + hot_layout_slot_masks() + .borrow_mut() + .insert(user_ptr, pointer_mask); } } diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index d7e9c6303d..d9c1db1086 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -48,6 +48,11 @@ mod telemetry; pub use telemetry::*; mod malloc; pub use malloc::*; +/// #7469: the `gc` half of the hot-thread-local address cache. Split out of +/// `barrier.rs` / `layout.rs` / `malloc.rs` so each stays under the repo's +/// 2000-line-per-file cap. +mod hot_tls; +pub(crate) use hot_tls::*; mod roots; pub use roots::*; /// #7148: the census of conservative-scan fallbacks and the precise-safepoint diff --git a/crates/perry-runtime/src/gc/roots.rs b/crates/perry-runtime/src/gc/roots.rs index a4e6ab190d..bcb2e07365 100644 --- a/crates/perry-runtime/src/gc/roots.rs +++ b/crates/perry-runtime/src/gc/roots.rs @@ -55,6 +55,8 @@ pub(crate) use shadow_stack::{shadow_stack_restore, shadow_stack_savepoint, Shad pub(crate) use temp_roots::reset_temp_roots; #[cfg(test)] pub(super) use temp_roots::temp_root_depth; +/// #7469: consumed by `crate::tls_hot::fill`, which lives outside `gc`. +pub(crate) use temp_roots::temp_roots_hot_addr; pub use temp_roots::{ js_array_push_f64_temp_rooted, js_gc_temp_root_get, js_gc_temp_root_push, js_gc_temp_root_set, js_gc_temp_root_truncate, diff --git a/crates/perry-runtime/src/gc/roots/temp_roots.rs b/crates/perry-runtime/src/gc/roots/temp_roots.rs index f19c957870..8405fdf8d9 100644 --- a/crates/perry-runtime/src/gc/roots/temp_roots.rs +++ b/crates/perry-runtime/src/gc/roots/temp_roots.rs @@ -60,6 +60,26 @@ thread_local! { std::cell::UnsafeCell::new(Vec::with_capacity(TEMP_ROOT_RESERVE)); } +// --- #7469 hot-TLS address provider. See `crate::tls_hot`. --- + +/// Address of this thread's `TEMP_ROOTS`. +pub(crate) fn temp_roots_hot_addr() -> *mut u8 { + TEMP_ROOTS.with(|cell| cell.get() as *mut u8) +} + +/// This thread's temp-root stack without a TLS resolution. +/// +/// `TEMP_ROOTS` is lazily initialised (its `Vec::with_capacity` is not const), +/// so every `.with()` pays `_tlv_get_addr` **plus** an initialised/destroyed +/// check. Generated code calls push/get/set/truncate as four separate FFI +/// calls around a single allocating expression, so that toll is paid four +/// times per temporary: 144 of the 653 attributed `_tlv_get_addr` samples on +/// `churn.ts`. +#[inline(always)] +fn hot_temp_roots() -> *mut Vec { + crate::tls_hot::hot().temp_roots as *mut Vec +} + /// Report a temp-root stack overflow without unwinding. /// /// Same defect class as #7145's `js_shadow_frame_pop`: a `debug_assert!(false)` @@ -85,8 +105,8 @@ a live slot. Reported once per process." /// `js_gc_temp_root_get` / `js_gc_temp_root_set` / `js_gc_temp_root_truncate`. #[no_mangle] pub extern "C" fn js_gc_temp_root_push(value: u64) -> u32 { - TEMP_ROOTS.with(|cell| unsafe { - let s = &mut *cell.get(); + unsafe { + let s = &mut *hot_temp_roots(); let idx = s.len(); // A depth this large means codegen dropped a truncate; refusing to grow // keeps a runaway from turning into unbounded retention. The returned @@ -100,44 +120,44 @@ pub extern "C" fn js_gc_temp_root_push(value: u64) -> u32 { crate::gc::runtime_write_barrier_root_heap_word(value); } idx as u32 - }) + } } /// Read slot `idx` back. Generated code must use this value, not the register /// it pushed: an evacuating cycle rewrites the slot in place. #[no_mangle] pub extern "C" fn js_gc_temp_root_get(idx: u32) -> u64 { - TEMP_ROOTS.with(|cell| unsafe { - let s = &*cell.get(); + unsafe { + let s = &*hot_temp_roots(); s.get(idx as usize).copied().unwrap_or(0) - }) + } } /// Overwrite slot `idx`, for producers that hand back a possibly-reallocated /// pointer (`js_array_push_f64`). #[no_mangle] pub extern "C" fn js_gc_temp_root_set(idx: u32, value: u64) { - TEMP_ROOTS.with(|cell| unsafe { - let s = &mut *cell.get(); + unsafe { + let s = &mut *hot_temp_roots(); if let Some(slot) = s.get_mut(idx as usize) { *slot = value; if value != 0 { crate::gc::runtime_write_barrier_root_heap_word(value); } } - }); + } } /// Drop slot `base` and every slot above it. #[no_mangle] pub extern "C" fn js_gc_temp_root_truncate(base: u32) { - TEMP_ROOTS.with(|cell| unsafe { - let s = &mut *cell.get(); + unsafe { + let s = &mut *hot_temp_roots(); let base = base as usize; if base < s.len() { s.truncate(base); } - }); + } } /// Push a rooted value onto the array in temp-root slot `idx`, writing the @@ -157,7 +177,7 @@ pub extern "C" fn js_array_push_f64_temp_rooted(idx: u32, value: f64) { /// Current depth — the value a savepoint records. pub(crate) fn temp_root_depth() -> usize { - TEMP_ROOTS.with(|cell| unsafe { (*cell.get()).len() }) + unsafe { (*hot_temp_roots()).len() } } /// Restore a previously-recorded depth. Used by the exception unwind path via diff --git a/crates/perry-runtime/src/lib.rs b/crates/perry-runtime/src/lib.rs index 86740c0d07..617314833e 100644 --- a/crates/perry-runtime/src/lib.rs +++ b/crates/perry-runtime/src/lib.rs @@ -149,6 +149,8 @@ pub mod temporal; pub(crate) mod test_support; pub mod text; pub mod timer; +/// #7469: one `_tlv_get_addr` for the whole allocation hot path. +pub(crate) mod tls_hot; pub mod typed_feedback; pub mod typedarray; pub mod typedarray_half; diff --git a/crates/perry-runtime/src/map.rs b/crates/perry-runtime/src/map.rs index 9379a702bb..b43a9d51f6 100644 --- a/crates/perry-runtime/src/map.rs +++ b/crates/perry-runtime/src/map.rs @@ -156,7 +156,31 @@ thread_local! { RefCell::new(crate::fast_hash::new_ptr_hash_map()); } +/// Has any thread ever registered a `Map`? +/// +/// Monotone — set at the one registration site below, never cleared, so it can +/// only ever be *conservatively* true. False proves this thread's +/// `MAP_REGISTRY` is empty, because a `Map` is only ever queried from the +/// thread that registered it (arenas are per-thread; values cross threads by +/// deep copy) and that thread's store precedes its own query in program order. +/// +/// #7469: `js_array_length` probes both this registry and the `Set` one on +/// every call — `arr.length` in a loop condition. On `churn.ts`, which creates +/// no `Map` and no `Set`, those two probes were 78 of the 520 remaining +/// `_tlv_get_addr` samples plus their hash cost, all to prove an empty map +/// stays empty. This turns both into a relaxed load of a static. +static MAP_REGISTRY_EVER_USED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +/// True when no `Map` has ever been registered, so `is_registered_map` can +/// answer without touching the thread-local registry. +#[inline(always)] +fn map_registry_never_used() -> bool { + !MAP_REGISTRY_EVER_USED.load(std::sync::atomic::Ordering::Relaxed) +} + fn register_map(ptr: *mut MapHeader, entries: *mut f64, capacity: usize) { + MAP_REGISTRY_EVER_USED.store(true, std::sync::atomic::Ordering::Relaxed); MAP_REGISTRY.with(|r| { let mut registry = r.borrow_mut(); assert!( @@ -168,6 +192,12 @@ fn register_map(ptr: *mut MapHeader, entries: *mut f64, capacity: usize) { } pub fn is_registered_map(addr: usize) -> bool { + // #7469: nothing has ever been registered ⟹ nothing can be found. Checked + // first because it is the only arm that costs neither a thread-local + // resolution nor a hash. + if map_registry_never_used() { + return false; + } // #4004: small-handle registry ids (Web Fetch, perry-ffi/node:http, timers, // …) are NaN-boxed POINTER_TAG values living below the small-handle // cutoff; they are not heap addresses. Managed Maps are arena-allocated diff --git a/crates/perry-runtime/src/set.rs b/crates/perry-runtime/src/set.rs index c32108d788..5e0d74de49 100644 --- a/crates/perry-runtime/src/set.rs +++ b/crates/perry-runtime/src/set.rs @@ -196,7 +196,21 @@ thread_local! { > = RefCell::new(crate::fast_hash::new_ptr_hash_map()); } +/// Has any thread ever registered a `Set`? Monotone twin of +/// `map::MAP_REGISTRY_EVER_USED` — see that flag for the full rationale and +/// the #7469 measurement. +static SET_REGISTRY_EVER_USED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +/// True when no `Set` has ever been registered, so `is_registered_set` can +/// answer without touching the thread-local registry. +#[inline(always)] +fn set_registry_never_used() -> bool { + !SET_REGISTRY_EVER_USED.load(std::sync::atomic::Ordering::Relaxed) +} + fn register_set(ptr: *mut SetHeader, elements: *mut f64, capacity: usize) { + SET_REGISTRY_EVER_USED.store(true, std::sync::atomic::Ordering::Relaxed); SET_REGISTRY.with(|r| { let mut registry = r.borrow_mut(); assert!( @@ -208,6 +222,11 @@ fn register_set(ptr: *mut SetHeader, elements: *mut f64, capacity: usize) { } pub fn is_registered_set(addr: usize) -> bool { + // #7469: nothing registered ⟹ nothing to find, without a thread-local + // resolution or a hash. See `map::is_registered_map` for the pairing. + if set_registry_never_used() { + return false; + } // #4004: reject the small-handle band (Web Fetch / node:http / timer ids // are NaN-boxed POINTER_TAG values, not heap addresses) before // dereferencing the GC header. Managed Sets are arena-allocated above the diff --git a/crates/perry-runtime/src/tls_hot.rs b/crates/perry-runtime/src/tls_hot.rs new file mode 100644 index 0000000000..da28344327 --- /dev/null +++ b/crates/perry-runtime/src/tls_hot.rs @@ -0,0 +1,274 @@ +//! One thread-local resolution for the whole allocation hot path (#7469). +//! +//! # Why this exists +//! +//! On Darwin every `thread_local!` access is an out-of-line call to +//! `_tlv_get_addr` in `libdyld`. Unlike ELF's `local-exec` / `initial-exec` +//! models it is a real call — not inlined, not cached across accesses, and it +//! clobbers caller-saved registers at the site. LLVM *can* CSE repeated +//! accesses to the **same** thread-local within a function, but two different +//! thread-locals are two different descriptors, so N distinct thread-locals on +//! one code path cost N calls no matter how well the path inlines. +//! +//! The runtime declares 237 `thread_local!` blocks, and a single +//! `{v, w}` object literal touches roughly a dozen of them: the arena and its +//! inline bump state, the free-list flag, the birth-flag cell, the layout side +//! tables, the page-generation cache the write barrier classifies against, and +//! the temp-root stack. Measured on `gc-handoff/bench/churn.ts` at +//! `351742d30`, `_tlv_get_addr` was **34.2% of all self time** — more than the +//! allocation work it was gating, and invisible to `PERRY_GC_TRACE` because it +//! is mutator time, not pause time. +//! +//! # What this does +//! +//! [`HotTls`] caches the **addresses** of those thread-locals in one +//! `const`-initialised thread-local. The storage does not move: every field is +//! the address of the existing `thread_local!` in its owning module, so +//! initialisation order, lazy init, and destructor registration are all +//! unchanged. A hot path that used to pay N `_tlv_get_addr` calls now pays one +//! (for `HOT` itself, which LLVM then CSEs across the whole inlined region) +//! plus N loads from one cache line. +//! +//! # Contract for adding a field +//! +//! 1. Add the `*mut u8` slot below. +//! 2. Add a `pub(crate) fn …_hot_addr() -> *mut u8` next to the `thread_local!` +//! that owns the storage, returning `KEY.with(|k| k as *const _ as *mut u8)`. +//! 3. Wire it in [`fill`]. +//! 4. Add the pair to `tls_hot::tests::cached_addresses_match_thread_locals`. +//! +//! Step 4 is the load-bearing one: the slots are untyped (`*mut u8`) so the +//! owning module can keep its storage type private, which means a mis-wired +//! `fill` would hand out a correctly-typed reference to the *wrong* object. +//! The test compares each cached address against the `.with()` address it is +//! supposed to mirror, so a mis-wire is a red build rather than a silent +//! cross-cast. +//! +//! # Lifetime +//! +//! The accessors hand out `&'static` references. That is sound for the *cache* +//! (const-init, no `Drop`, so it is never destroyed) and carries exactly the +//! same thread-teardown exposure the runtime already has for `ARENA` and +//! `INLINE_STATE`, whose raw pointers are handed to generated code by +//! `js_inline_arena_state`. It is not an invitation to send one across +//! threads, and the pointee types (`Cell`, `RefCell`, `UnsafeCell`) are all +//! `!Sync`, so the compiler refuses that on its own. + +use std::cell::UnsafeCell; + +/// Cached addresses of the per-thread state on the allocation hot path. +/// +/// Slots are untyped so each owning module keeps its storage type private; +/// the typed accessor lives next to the `thread_local!` it casts back to. +#[repr(C)] +pub(crate) struct HotTls { + // arena/block.rs + pub(crate) arena: *mut u8, + pub(crate) inline_state: *mut u8, + // arena/page_meta.rs + pub(crate) page_generation_cache: *mut u8, + pub(crate) page_generations: *mut u8, + // gc/malloc.rs + pub(crate) arena_free_list: *mut u8, + pub(crate) arena_free_list_nonempty: *mut u8, + // gc/barrier.rs + pub(crate) birth_extra_flags: *mut u8, + pub(crate) incremental_mark_valid_ptrs: *mut u8, + pub(crate) incremental_mark_minor_only: *mut u8, + // gc/layout.rs + pub(crate) layout_slot_masks: *mut u8, + pub(crate) typed_layouts: *mut u8, + pub(crate) shape_layouts: *mut u8, + // gc/roots/temp_roots.rs + pub(crate) temp_roots: *mut u8, +} + +impl HotTls { + const EMPTY: Self = Self { + arena: std::ptr::null_mut(), + inline_state: std::ptr::null_mut(), + page_generation_cache: std::ptr::null_mut(), + page_generations: std::ptr::null_mut(), + arena_free_list: std::ptr::null_mut(), + arena_free_list_nonempty: std::ptr::null_mut(), + birth_extra_flags: std::ptr::null_mut(), + incremental_mark_valid_ptrs: std::ptr::null_mut(), + incremental_mark_minor_only: std::ptr::null_mut(), + layout_slot_masks: std::ptr::null_mut(), + typed_layouts: std::ptr::null_mut(), + shape_layouts: std::ptr::null_mut(), + temp_roots: std::ptr::null_mut(), + }; +} + +thread_local! { + /// `const`-initialised on purpose: a lazily-initialised `thread_local!` + /// pays a "has this been initialised / has this been dropped" check on + /// every `.with()` **on top of** `_tlv_get_addr`, and registers a + /// destructor. `HotTls` is plain pointers with no `Drop`, so the const + /// form reduces the one remaining resolution to the bare thunk call. + static HOT: UnsafeCell = const { UnsafeCell::new(HotTls::EMPTY) }; +} + +/// Resolve every cached address for this thread. Cold: runs once per thread. +/// +/// Each `…_hot_addr()` touches its own `thread_local!` exactly as any other +/// caller would, so a lazily-initialised one is initialised here instead of at +/// its first hot-path use. That is a move in when, not in what. +#[cold] +#[inline(never)] +fn fill(slots: *mut HotTls) { + // SAFETY: `slots` is this thread's own cache; no other thread can observe + // it and the runtime is single-threaded per arena. + unsafe { + (*slots).arena = crate::arena::arena_hot_addr(); + (*slots).inline_state = crate::arena::inline_state_hot_addr(); + (*slots).page_generation_cache = crate::arena::page_generation_cache_hot_addr(); + (*slots).page_generations = crate::arena::page_generations_hot_addr(); + (*slots).arena_free_list = crate::gc::arena_free_list_hot_addr(); + (*slots).arena_free_list_nonempty = crate::gc::arena_free_list_nonempty_hot_addr(); + (*slots).birth_extra_flags = crate::gc::birth_extra_flags_hot_addr(); + (*slots).incremental_mark_valid_ptrs = crate::gc::incremental_mark_valid_ptrs_hot_addr(); + (*slots).incremental_mark_minor_only = crate::gc::incremental_mark_minor_only_hot_addr(); + (*slots).layout_slot_masks = crate::gc::layout_slot_masks_hot_addr(); + (*slots).typed_layouts = crate::gc::typed_layouts_hot_addr(); + (*slots).shape_layouts = crate::gc::shape_layouts_hot_addr(); + // Last, and the field `hot()` tests: every other slot is already + // written by the time this one is non-null, so a re-entrant call from + // inside one of the providers above cannot observe a half-filled cache + // as ready. + (*slots).temp_roots = crate::gc::temp_roots_hot_addr(); + } +} + +/// The per-thread address cache. One `_tlv_get_addr` for every thread-local it +/// covers. +#[inline(always)] +pub(crate) fn hot() -> &'static HotTls { + let slots = HOT.with(|cell| cell.get()); + // SAFETY: `HOT` is const-init with no `Drop`, so its storage is valid for + // the whole life of the thread — see the module docs on lifetime. + unsafe { + if (*slots).temp_roots.is_null() { + fill(slots); + } + &*slots + } +} + +#[cfg(test)] +mod tests { + /// Every cached address must equal the address of the `thread_local!` it + /// mirrors. The slots are untyped, so this is what stands between a + /// mis-wired [`super::fill`] and a well-typed reference to the wrong + /// object. + #[test] + fn cached_addresses_match_thread_locals() { + let hot = super::hot(); + assert_eq!(hot.arena, crate::arena::arena_hot_addr(), "arena"); + assert_eq!( + hot.inline_state, + crate::arena::inline_state_hot_addr(), + "inline_state" + ); + assert_eq!( + hot.page_generation_cache, + crate::arena::page_generation_cache_hot_addr(), + "page_generation_cache" + ); + assert_eq!( + hot.page_generations, + crate::arena::page_generations_hot_addr(), + "page_generations" + ); + assert_eq!( + hot.arena_free_list, + crate::gc::arena_free_list_hot_addr(), + "arena_free_list" + ); + assert_eq!( + hot.arena_free_list_nonempty, + crate::gc::arena_free_list_nonempty_hot_addr(), + "arena_free_list_nonempty" + ); + assert_eq!( + hot.birth_extra_flags, + crate::gc::birth_extra_flags_hot_addr(), + "birth_extra_flags" + ); + assert_eq!( + hot.incremental_mark_valid_ptrs, + crate::gc::incremental_mark_valid_ptrs_hot_addr(), + "incremental_mark_valid_ptrs" + ); + assert_eq!( + hot.incremental_mark_minor_only, + crate::gc::incremental_mark_minor_only_hot_addr(), + "incremental_mark_minor_only" + ); + assert_eq!( + hot.layout_slot_masks, + crate::gc::layout_slot_masks_hot_addr(), + "layout_slot_masks" + ); + assert_eq!( + hot.typed_layouts, + crate::gc::typed_layouts_hot_addr(), + "typed_layouts" + ); + assert_eq!( + hot.shape_layouts, + crate::gc::shape_layouts_hot_addr(), + "shape_layouts" + ); + assert_eq!( + hot.temp_roots, + crate::gc::temp_roots_hot_addr(), + "temp_roots" + ); + } + + /// No slot may be null after `hot()` — a null would mean `fill` skipped a + /// provider, and the typed accessor would dereference it. + #[test] + fn every_slot_is_populated() { + let hot = super::hot(); + for (name, ptr) in [ + ("arena", hot.arena), + ("inline_state", hot.inline_state), + ("page_generation_cache", hot.page_generation_cache), + ("page_generations", hot.page_generations), + ("arena_free_list", hot.arena_free_list), + ("arena_free_list_nonempty", hot.arena_free_list_nonempty), + ("birth_extra_flags", hot.birth_extra_flags), + ( + "incremental_mark_valid_ptrs", + hot.incremental_mark_valid_ptrs, + ), + ( + "incremental_mark_minor_only", + hot.incremental_mark_minor_only, + ), + ("layout_slot_masks", hot.layout_slot_masks), + ("typed_layouts", hot.typed_layouts), + ("shape_layouts", hot.shape_layouts), + ("temp_roots", hot.temp_roots), + ] { + assert!(!ptr.is_null(), "{name} slot was left null by fill()"); + } + } + + /// The cache is per-thread: a second thread must resolve its own + /// addresses, not inherit this one's. + #[test] + fn each_thread_caches_its_own_addresses() { + let mine = super::hot().temp_roots as usize; + let theirs = std::thread::spawn(|| super::hot().temp_roots as usize) + .join() + .expect("probe thread panicked"); + assert_ne!( + mine, theirs, + "two threads resolved the same temp-root address" + ); + } +} From 20b716f16293abf7ac497dcafbbdd31f419cf0de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 6 Aug 2026 01:26:23 +0200 Subject: [PATCH 2/2] docs(changelog): fragment for the #7469 hot-TLS allocation-path work --- changelog.d/7474-hot-tls-alloc-path.md | 90 ++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 changelog.d/7474-hot-tls-alloc-path.md diff --git a/changelog.d/7474-hot-tls-alloc-path.md b/changelog.d/7474-hot-tls-alloc-path.md new file mode 100644 index 0000000000..b84c78b995 --- /dev/null +++ b/changelog.d/7474-hot-tls-alloc-path.md @@ -0,0 +1,90 @@ +### Allocation path: one `_tlv_get_addr` per runtime call instead of a dozen (#7469) + +With the collector no longer the limit on allocation-heavy programs (total GC +pause on `gc-handoff/bench/churn.ts` is 0.03 s of a 4.3 s run), the cost moved +into the mutator's allocation path — and 27.9% of self time there was +`_tlv_get_addr`, macOS's thread-local accessor. + +On Darwin every `thread_local!` access is an out-of-line call into `libdyld`. +Unlike ELF's `local-exec` / `initial-exec` models it is neither inlined nor +cached across accesses, and — the part that makes it add up — LLVM can CSE +repeated accesses to the *same* thread-local, but two different thread-locals +are two different descriptors. So N distinct thread-locals on one code path cost +N calls however well that path inlines. A single `{v, w}` object literal touched +about a dozen: the arena and its inline bump state, the free-list flag, the +allocate-black birth flags, three layout side tables, the page-generation cache +the write barrier classifies against, and the temp-root stack. + +`crates/perry-runtime/src/tls_hot.rs` caches those *addresses* in one +`const`-initialised thread-local. The storage does not move: every slot holds +the address of the existing `thread_local!` in its owning module, so +initialisation order, lazy init and destructor registration are unchanged. The +slots are untyped (`*mut u8`) so each owning module keeps its storage type +private, which means a mis-wired `fill()` would hand out a *well-typed* +reference to the wrong object — +`tls_hot::tests::cached_addresses_match_thread_locals` asserts every +address/accessor pairing, and `every_slot_is_populated` asserts nothing was +skipped. + +Profiling the remainder (symbolicated `sample`, attributing each +`_tlv_get_addr` to its immediate caller) turned up four more things: + +- **`incremental_mark_barrier_value` read a thread-local on every heap-pointer + store** — 91 of 653 attributed samples, every one spent proving a null pointer + was still null. It now consults the process-global + `PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT` first, the same authority + generated code already trusts for this exact question. Consequently + `incremental_mark_barrier_enable` now arms that count **before** installing + the thread-local pointer (`disable` already cleared the pointer before + decrementing), so no window exists where the pointer is live while the count + reads idle. That ordering used to be merely tidy; a store landing in such a + window would now skip its insertion barrier, i.e. lose a mark, so the + requirement is written down at the site. +- **`js_array_length` probed the `Map` and `Set` registries on every call** — an + `arr.length` loop condition paying two thread-local hash lookups. Monotone + "has anything ever been registered" flags answer for programs that use + neither. Monotone on purpose: a maintained count is a count that can be got + wrong across eight mutation sites, and this only ever needs to prove that + *nothing exists*. +- **The page-generation cache held one entry**, while the write barrier + classifies at least two unrelated addresses per store (the child written, then + the parent written into). On `churn.ts` those sit in different 1 MiB + generation classes, so consecutive classifications evicted each other and the + cache missed on essentially every call — 71 self samples in the authoritative + map lookup it exists to avoid. Now 4-way, and behind an `UnsafeCell` rather + than a `Cell`: `Cell::get` returns a *copy*, and copying the set per + classification cost more than the lookup (caught as a 2% `retain.ts` + regression while bisecting the arms of this change). +- **`layout_forget_object`** takes one `borrow_mut` per map instead of a + `borrow` to test emptiness followed by a second `borrow_mut` to remove. + +`gc/hot_tls.rs` holds the `gc`-side accessor/provider pairs, split out because +`barrier.rs` crossed the 2000-line cap; keeping each pair together is what makes +the untyped casts reviewable. + +Measured on `gc-handoff/bench`, arms interleaved round by round so machine-load +drift hits both equally, best-of-5 user CPU on a quiet host: churn **1.16x**, +cycles **1.10x**, retain1 **1.08x**, retain **1.07x**, deeplist **1.07x**, tree +**1.03x**. Peak RSS flat (worst cell +0.9%), program output byte-identical on +all six. `_tlv_get_addr` falls to 23.5% of leaf samples; attributed callers 653 +→ 542. + +The collector is untouched, and that is checked rather than assumed. The pinned +`gc_ratchet` artifact currently fails on `main` itself (28 regressions predating +#7432/#7443/#7449), so clean `main` was measured with the same harness on the +same host and diffed: all 108 compared metrics agree except `heap_used_bytes` on +2 of 12 probes at ≤0.8%, which moves on *different* probes per build and is +therefore allocation-boundary jitter. Every metric in the `gc` family +(`minor_cycles`, `step_cycles`, `copied_objects`, `copied_bytes`, +`promoted_objects`, `promoted_bytes`, `freed_bytes`) is identical across all 12 +probes. Per-cycle `PERRY_GC_TRACE` is unchanged on churn (105 cycles, 0.03 s +total pause), tree (43) and retain (11), with tree copying volume holding at the +post-#7432 level of 0.017 GB / 0.2 M object-copies. + +This does not close #7469. The floor of this design is *one* resolution per +runtime FFI call, and generated code makes roughly a dozen per object literal — +`js_gc_temp_root_push` / `_get` / `_truncate` alone are 206 of the 542 remaining +samples, three separate calls around one allocating expression. Getting under +the ticket's 5% target needs the call *count* to fall, which is workstream A +item 4 (codegen emitting the bump allocation inline). Workstream B (per-object +footprint) is untouched here.