diff --git a/changelog.d/7443-oldgen-hole-free-list.md b/changelog.d/7443-oldgen-hole-free-list.md new file mode 100644 index 0000000000..08554ee4f1 --- /dev/null +++ b/changelog.d/7443-oldgen-hole-free-list.md @@ -0,0 +1,5 @@ +**GC: old-generation hole free list** (#7437) — old-gen allocation was pure bump, so a swept dead old object stayed dead capacity until its *entire* block died; a block with one live object never reset. Scattered survivors therefore pinned 105.6 MB of blocks for a ~1 MB live set (the `12_large_live_set` ratchet probe: a full collection freed 87.6 MB and reclaimed nothing, 49/50 blocks live), and the dead bytes kept counting as old-gen pressure, re-firing full collections that could not lower the number they watched. + +Swept dead old objects in still-live blocks now become exact-size reusable holes (`gc/old_free.rs`): a size-bucketed map rebuilt at each old-reclaiming sweep's completion from a raw-headers walk over surviving old blocks (invalidated dead headers are exactly `obj_type == 0`; the walkable-gated walkers skip them without invoking the callback, so the raw walker `old_arena_walk_all_headers_filtered` is load-bearing). `arena_alloc_gc_old` (promotions, large-object births) and its defrag-aware variant consume holes through the standard birth path; old-block reset/dealloc sites filter their ranges; the old-reclaim pacing arms and `process.memoryUsage().heapUsed` subtract the reusable bytes. Exact fit keeps `GcHeader::size` in agreement with per-object promotion accounting. `PERRY_GC_DIAG=1` prints `[gc-old-free] reusable_bytes=` after each rebuild. + +Measured: probe 12 retained `heapUsed` after the release-phase `gc()` drops 105.6 MB → 59.9 MB (the residual is longlived-arena and young keep-window bytes, not old-gen) at ±1% peak RSS; mid-run the reusable pool cycles 2.1 → 8.4 → 45.4 MB as holes are consumed by later promotions; all five GC benchmark traces are byte-identical. Also splits old-page defrag selection into `gc/oldgen_defrag.rs` (2000-line lint cap). diff --git a/crates/perry-runtime/src/arena/allocators.rs b/crates/perry-runtime/src/arena/allocators.rs index ae287006ca..6c228c6e45 100644 --- a/crates/perry-runtime/src/arena/allocators.rs +++ b/crates/perry-runtime/src/arena/allocators.rs @@ -121,6 +121,23 @@ pub fn arena_alloc_gc_old(size: usize, align: usize, obj_type: u8) -> *mut u8 { // Same alignment-preservation rationale as `arena_alloc_gc`. let pad = align.max(8); let total = (GC_HEADER_SIZE + size + pad - 1) & !(pad - 1); + // #7437: reuse a swept same-size hole before bumping — otherwise a + // block with any live object never yields its dead bytes back and old + // capacity only ever grows. Exact fit keeps `GcHeader::size` equal to + // what per-object promotion accounting records for this allocation. + if let Some(user_ptr) = crate::gc::old_free_take_exact(total, None) { + let raw = (user_ptr - GC_HEADER_SIZE) as *mut u8; + unsafe { + let header = raw as *mut GcHeader; + (*header).obj_type = obj_type; + (*header).gc_flags = GC_FLAG_ARENA | crate::gc::gc_birth_extra_flags(); + crate::gc::gc_note_black_birth(header); + (*header)._reserved = 0; + (*header).size = total as u32; + } + register_old_object_pages(raw as usize, total); + return user_ptr as *mut u8; + } let raw = arena_alloc_old(total, align); unsafe { @@ -146,6 +163,21 @@ pub(crate) fn arena_alloc_gc_old_excluding_pages( let pad = align.max(8); let total = (GC_HEADER_SIZE + size + pad - 1) & !(pad - 1); + // #7437: same hole reuse as `arena_alloc_gc_old`, but never into a page + // this defrag pass is evacuating. + if let Some(user_ptr) = crate::gc::old_free_take_exact(total, Some(excluded_pages)) { + let raw = (user_ptr - GC_HEADER_SIZE) as *mut u8; + unsafe { + let header = raw as *mut GcHeader; + (*header).obj_type = obj_type; + (*header).gc_flags = GC_FLAG_ARENA | crate::gc::gc_birth_extra_flags(); + crate::gc::gc_note_black_birth(header); + (*header)._reserved = 0; + (*header).size = total as u32; + } + register_old_object_pages(raw as usize, total); + return user_ptr as *mut u8; + } let raw = arena_alloc_old_excluding_pages(total, align, excluded_pages); unsafe { diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index 80cb3a1c6e..4480e882ae 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -70,8 +70,8 @@ pub use walk::{ }; pub(crate) use walk::{ arena_block_snapshots, arena_telemetry_snapshot, general_block_in_recent_window, - general_block_sizes, ArenaBlockSnapshot, ArenaObjectCursor, ArenaObjectCursorBuilder, - ArenaTelemetrySnapshot, ArenaWalkOrder, + general_block_sizes, old_arena_walk_all_headers_filtered, ArenaBlockSnapshot, + ArenaObjectCursor, ArenaObjectCursorBuilder, ArenaTelemetrySnapshot, ArenaWalkOrder, }; // reset.rs diff --git a/crates/perry-runtime/src/arena/reset.rs b/crates/perry-runtime/src/arena/reset.rs index d981ce6d2f..dd11c5e153 100644 --- a/crates/perry-runtime/src/arena/reset.rs +++ b/crates/perry-runtime/src/arena/reset.rs @@ -1025,6 +1025,9 @@ impl OldArenaReclaimDeadBlocksState { let last_page = generation_page_for_addr(base + size - 1); let pages: Vec = (first_page..=last_page).collect(); unregister_old_block_pages(&pages); + // #7437: this block's bytes are being recycled; any swept hole + // recorded inside it must not be handed out again. + crate::gc::old_free_filter_range(base, size); if used != 0 { self.stats.reset_blocks = self.stats.reset_blocks.saturating_add(1); @@ -1115,6 +1118,9 @@ pub(crate) fn old_arena_reclaim_dead_blocks(block_has_live: &[bool]) -> ArenaRes let last_page = generation_page_for_addr(base + size - 1); let pages: Vec = (first_page..=last_page).collect(); unregister_old_block_pages(&pages); + // #7437: this block's bytes are being recycled; any swept hole + // recorded inside it must not be handed out again. + crate::gc::old_free_filter_range(base, size); if used != 0 { stats.reset_blocks = stats.reset_blocks.saturating_add(1); @@ -1217,6 +1223,9 @@ pub(crate) fn old_arena_reclaim_selected_dead_blocks( let last_page = generation_page_for_addr(base + size - 1); let pages: Vec = (first_page..=last_page).collect(); unregister_old_block_pages(&pages); + // #7437: this block's bytes are being recycled; any swept hole + // recorded inside it must not be handed out again. + crate::gc::old_free_filter_range(base, size); if used != 0 { stats.reset_blocks = stats.reset_blocks.saturating_add(1); diff --git a/crates/perry-runtime/src/arena/stats.rs b/crates/perry-runtime/src/arena/stats.rs index 5494218462..973c2ee085 100644 --- a/crates/perry-runtime/src/arena/stats.rs +++ b/crates/perry-runtime/src/arena/stats.rs @@ -50,6 +50,13 @@ pub extern "C" fn js_arena_stats(out_used: *mut u64, out_total: *mut u64) { total += block.size as u64; } }); + // #7437: swept old-gen holes are reusable capacity, not used heap. + // Block offsets cannot express a hole (the bump pointer never moves + // back), so without this subtraction `heapUsed` reports the scattered- + // survivor high-water forever — 105.6 MB for a ~1 MB live set on the + // 12_large_live_set ratchet probe — and looks like a leak that no + // amount of collecting can fix. + used = used.saturating_sub(crate::gc::old_free_bytes() as u64); unsafe { *out_used = used; *out_total = total; diff --git a/crates/perry-runtime/src/arena/walk.rs b/crates/perry-runtime/src/arena/walk.rs index e9c470bc29..072cfacee8 100644 --- a/crates/perry-runtime/src/arena/walk.rs +++ b/crates/perry-runtime/src/arena/walk.rs @@ -839,3 +839,43 @@ pub fn longlived_end() -> usize { let l = LONGLIVED_ARENA.with(|arena| unsafe { (*arena.get()).blocks.len() }); g + s0 + s1 + l } + +/// #7437: walk EVERY header in the selected old-gen blocks, including +/// invalidated dead ones (`obj_type == 0`), which the walkable-gated +/// walkers above deliberately skip. Stepping is by `GcHeader::size`, which +/// `invalidate_dead_old_arena_header` preserves exactly so holes remain +/// traversable. `block_filter` receives GLOBAL block indices (same base as +/// `arena_walk_objects_filtered`'s old-gen region). +pub(crate) fn old_arena_walk_all_headers_filtered( + mut block_filter: impl FnMut(usize) -> bool, + mut callback: impl FnMut(*mut u8, usize), +) { + use crate::gc::GcHeader; + let old_block_start = longlived_end(); + OLD_ARENA.with(|arena| { + let arena = unsafe { &*arena.get() }; + for (i, block) in arena.blocks.iter().enumerate() { + let block_idx = old_block_start + i; + if block.data.is_null() || !block_filter(block_idx) { + continue; + } + let mut offset = 0usize; + while offset < block.offset { + let aligned = (offset + 7) & !7; + if aligned >= block.offset { + break; + } + let header_ptr = unsafe { block.data.add(aligned) }; + let header = header_ptr as *const GcHeader; + unsafe { + let total_size = (*header).size as usize; + if total_size == 0 || total_size > block.size { + break; + } + callback(header_ptr, block_idx); + offset = aligned + total_size; + } + } + } + }); +} diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index b4ef765356..f5b438f19e 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -83,10 +83,15 @@ use copying::*; // pass in `crate::weakref` (#6182), which lives outside the gc module. pub(crate) use copying::CopyingPointerSet; mod dead_owner; +mod old_free; +use old_free::*; +pub(crate) use old_free::{old_free_bytes, old_free_filter_range, old_free_take_exact}; mod tenuring; use tenuring::*; mod oldgen; use oldgen::*; +mod oldgen_defrag; +use oldgen_defrag::*; mod cycle; use cycle::*; mod verify; diff --git a/crates/perry-runtime/src/gc/old_free.rs b/crates/perry-runtime/src/gc/old_free.rs new file mode 100644 index 0000000000..ea26c3e4d7 --- /dev/null +++ b/crates/perry-runtime/src/gc/old_free.rs @@ -0,0 +1,193 @@ +//! Old-generation hole free list (#7437). +//! +//! Old-gen allocation was pure bump: a swept dead old object stayed dead +//! capacity until its *entire block* died, and a block with even one live +//! object never resets. A workload that promotes a large cohort and keeps +//! a scattered subset (every-64th node in the `12_large_live_set` ratchet +//! probe) therefore retained 105 MB of blocks for a ~1 MB live set — the +//! final full collection freed 87 MB of objects and reclaimed nothing, +//! because 49 of 50 blocks still held at least one live object. The same +//! mechanism is a large slice of tree.ts's old-gen churn high-water +//! (#7438): every dropped tree leaves holes in blocks pinned live by the +//! next tree's nodes. +//! +//! This module gives the old generation what the general arena has had +//! all along (`ARENA_FREE_LIST`): swept holes become reusable. Shape +//! differences are deliberate: +//! +//! - **Exact fit only, keyed by total (header-inclusive, padded) size.** +//! The general list best-fits into larger slots and keeps the slot's +//! original `GcHeader::size`, which is fine there because nothing else +//! accounts those bytes. Old-gen promotion *does* account per-object +//! sizes (`old_page_account_promoted_object`), so a reused slot must +//! have exactly the size the caller asked for or the page live-byte +//! accounting diverges from the header. Promoted cohorts are dominated +//! by uniform class-instance sizes, so exact fit hits where the +//! pathology lives. +//! - **Size-bucketed map, not a scanned Vec.** The pathological case has +//! hundreds of thousands of holes; a per-allocation linear scan would +//! put an O(holes) tax on every promotion. +//! +//! `OLD_ARENA_FREE_BYTES` tracks the total. It is deliberately NOT +//! subtracted from `OLD_GEN_IN_USE_BYTES` — that cache is defined (and +//! debug-asserted) as the sum of old block offsets, which hole reuse does +//! not change. Consumers that want *live* old pressure (the old-reclaim +//! pacing arms, `process.memoryUsage().heapUsed`) subtract +//! [`old_free_bytes`] instead; before this, dead-but-unreclaimable bytes +//! counted as pressure, so old-reclaim kept re-firing full collections +//! that could not actually lower the number they were watching. +//! +//! Entries are only pushed for dead objects in blocks that still hold a +//! live object (fully-dead blocks go through block reclaim, which is +//! strictly better). A pushed entry's block can still die on a LATER +//! cycle, so every old-block reset/dealloc site must call +//! [`old_free_filter_range`] for the range it is about to recycle. + +use super::*; + +thread_local! { + /// total_size -> user_ptrs of swept holes of exactly that size. + static OLD_FREE_MAP: RefCell>> = + RefCell::new(crate::fast_hash::new_ptr_hash_map()); + static OLD_FREE_BYTES: Cell = const { Cell::new(0) }; + static OLD_FREE_NONEMPTY: Cell = const { Cell::new(false) }; +} + +/// Total bytes currently sitting in reusable old-gen holes. +pub(crate) fn old_free_bytes() -> usize { + OLD_FREE_BYTES.with(Cell::get) +} + +fn old_free_push(user_ptr: usize, total_size: usize) { + if user_ptr == 0 || total_size < GC_HEADER_SIZE { + return; + } + OLD_FREE_MAP.with(|m| { + m.borrow_mut().entry(total_size).or_default().push(user_ptr); + }); + OLD_FREE_BYTES.with(|c| c.set(c.get().saturating_add(total_size))); + OLD_FREE_NONEMPTY.with(|c| c.set(true)); +} + +/// Rebuild the hole map from the heap itself: every invalidated dead +/// header (`obj_type == 0` — only `invalidate_dead_old_arena_header` +/// produces those; no live object has type 0) inside an old block that +/// still holds a live object. Called at the completion point of every +/// old-reclaiming sweep, replacing whatever the map held. +/// +/// Rebuilding beats accumulating a staging vector during the sweep walk on +/// two counts, both measured on `12_large_live_set` (~700k dead old +/// objects): the staging vector alone added ~17 MB of peak RSS to the very +/// number this feature exists to lower, and rebuild is idempotent — a hole +/// consumed by reuse gets a real `obj_type` and drops out, a hole whose +/// block died is never visited, so no cross-sweep dedup bookkeeping can +/// drift. The walk is block-filtered (live old blocks only), so its cost +/// is O(objects in surviving old blocks), paid only on reclaim sweeps. +pub(super) fn old_free_rebuild_from_live_old_blocks( + block_has_live: &[bool], + old_block_start: usize, +) { + OLD_FREE_MAP.with(|m| m.borrow_mut().clear()); + OLD_FREE_BYTES.with(|c| c.set(0)); + OLD_FREE_NONEMPTY.with(|c| c.set(false)); + // The raw-headers walker is load-bearing: the walkable-gated walkers + // (`arena_walk_objects_filtered` and friends) step over invalidated + // headers WITHOUT invoking the callback, so a rebuild written against + // them silently records zero holes. + crate::arena::old_arena_walk_all_headers_filtered( + |block_idx| { + block_idx >= old_block_start && block_has_live.get(block_idx).copied().unwrap_or(false) + }, + |header_ptr, _block_idx| { + let header = header_ptr as *mut GcHeader; + unsafe { + if (*header).obj_type == 0 { + let total_size = (*header).size as usize; + old_free_push(header as usize + GC_HEADER_SIZE, total_size); + } + } + }, + ); +} + +/// Take a hole of exactly `total_size` bytes, if one exists. When +/// `excluded_pages` is non-empty the caller is mid-defrag and must not +/// allocate on the pages it is evacuating; holes on those pages are +/// skipped (and retained). +pub(crate) fn old_free_take_exact( + total_size: usize, + excluded_pages: Option<&crate::fast_hash::PtrHashSet>, +) -> Option { + if !OLD_FREE_NONEMPTY.with(Cell::get) { + return None; + } + let taken = OLD_FREE_MAP.with(|m| { + let mut map = m.borrow_mut(); + let bucket = map.get_mut(&total_size)?; + let taken = match excluded_pages { + None => bucket.pop(), + Some(excluded) => { + let idx = bucket.iter().rposition(|&ptr| { + let header = ptr - GC_HEADER_SIZE; + let first = crate::arena::generation_page_for_addr(header); + let last = crate::arena::generation_page_for_addr(header + total_size - 1); + (first..=last).all(|page| !excluded.contains(&page)) + })?; + Some(bucket.swap_remove(idx)) + } + }; + if bucket.is_empty() { + map.remove(&total_size); + } + taken + })?; + OLD_FREE_BYTES.with(|c| c.set(c.get().saturating_sub(total_size))); + OLD_FREE_MAP.with(|m| { + if m.borrow().is_empty() { + OLD_FREE_NONEMPTY.with(|c| c.set(false)); + } + }); + Some(taken) +} + +/// Drop every hole inside `[base, base + size)`. Called by the old-block +/// reset/dealloc paths before they recycle a block's bytes — a stale +/// entry would otherwise hand out a pointer into memory the bump +/// allocator is about to overwrite (or that has been returned to the OS). +pub(crate) fn old_free_filter_range(base: usize, size: usize) { + if !OLD_FREE_NONEMPTY.with(Cell::get) || size == 0 { + return; + } + let end = base.saturating_add(size); + let mut removed_bytes = 0usize; + OLD_FREE_MAP.with(|m| { + let mut map = m.borrow_mut(); + map.retain(|&slot_size, bucket| { + bucket.retain(|&ptr| { + let header = ptr - GC_HEADER_SIZE; + let inside = header >= base && header < end; + if inside { + removed_bytes = removed_bytes.saturating_add(slot_size); + } + !inside + }); + !bucket.is_empty() + }); + if map.is_empty() { + OLD_FREE_NONEMPTY.with(|c| c.set(false)); + } + }); + OLD_FREE_BYTES.with(|c| c.set(c.get().saturating_sub(removed_bytes))); +} + +#[cfg(test)] +pub(super) fn old_free_reset_for_test() { + OLD_FREE_MAP.with(|m| m.borrow_mut().clear()); + OLD_FREE_BYTES.with(|c| c.set(0)); + OLD_FREE_NONEMPTY.with(|c| c.set(false)); +} + +#[cfg(test)] +pub(super) fn old_free_entry_count() -> usize { + OLD_FREE_MAP.with(|m| m.borrow().values().map(|b| b.len()).sum()) +} diff --git a/crates/perry-runtime/src/gc/oldgen.rs b/crates/perry-runtime/src/gc/oldgen.rs index b145941258..6aae75062f 100644 --- a/crates/perry-runtime/src/gc/oldgen.rs +++ b/crates/perry-runtime/src/gc/oldgen.rs @@ -82,21 +82,6 @@ impl EvacuationPolicySnapshot { } } -#[derive(Default)] -pub(super) struct OldPageDefragSelection { - pub(super) pages: crate::fast_hash::PtrHashSet, - pub(super) page_order: Vec, - pub(super) candidate_pages: usize, - pub(super) selected_pages: usize, - pub(super) selected_live_bytes: usize, - pub(super) selected_reclaimable_bytes: usize, - /// Page-granule bytes the selected pages would hand back once their - /// movable live objects are evacuated: page size minus pinned bytes - /// (selection skips pinned pages, so in practice the full granule). - pub(super) selected_releasable_block_bytes: usize, - pub(super) skipped_pinned_pages: usize, -} - #[derive(Clone, Copy)] pub(super) struct EvacuationPolicyDecision { pub(super) allowed: bool, @@ -136,130 +121,6 @@ pub(super) struct SweepTraceStats { pub(super) retained_forwarded_stub_bytes: usize, } -#[inline] -pub(super) fn old_page_defrag_eligible(meta: crate::arena::OldPageMeta) -> bool { - meta.allocated_bytes > 0 && meta.live_bytes > 0 && meta.dead_bytes > 0 && meta.pinned_bytes == 0 -} - -#[inline] -pub(super) fn old_page_defrag_skipped_for_pin(meta: crate::arena::OldPageMeta) -> bool { - meta.allocated_bytes > 0 && meta.live_bytes > 0 && meta.dead_bytes > 0 && meta.pinned_bytes > 0 -} - -pub(super) fn select_old_page_defrag_pages_from_snapshot( - snapshot: &[crate::arena::OldPageMeta], - force: bool, -) -> OldPageDefragSelection { - let mut selection = OldPageDefragSelection::default(); - let mut candidates = Vec::new(); - for &meta in snapshot { - if old_page_defrag_skipped_for_pin(meta) { - selection.skipped_pinned_pages = selection.skipped_pinned_pages.saturating_add(1); - continue; - } - if !old_page_defrag_eligible(meta) { - continue; - } - selection.candidate_pages = selection.candidate_pages.saturating_add(1); - if force || meta.dead_bytes >= meta.live_bytes { - candidates.push(meta); - } - } - - candidates.sort_unstable_by(|a, b| { - let b_ratio = (b.dead_bytes as u128).saturating_mul(a.allocated_bytes as u128); - let a_ratio = (a.dead_bytes as u128).saturating_mul(b.allocated_bytes as u128); - b_ratio - .cmp(&a_ratio) - .then_with(|| a.live_bytes.cmp(&b.live_bytes)) - .then_with(|| a.page_base.cmp(&b.page_base)) - }); - - for meta in candidates { - let page = crate::arena::generation_page_for_addr(meta.page_base); - if selection.pages.insert(page) { - selection.page_order.push(page); - selection.selected_pages = selection.selected_pages.saturating_add(1); - selection.selected_live_bytes = selection - .selected_live_bytes - .saturating_add(meta.live_bytes); - selection.selected_reclaimable_bytes = selection - .selected_reclaimable_bytes - .saturating_add(meta.dead_bytes); - selection.selected_releasable_block_bytes = - selection.selected_releasable_block_bytes.saturating_add( - (meta.page_end.saturating_sub(meta.page_base)) - .saturating_sub(meta.pinned_bytes), - ); - } - } - - selection -} - -// gh #6206 test hook: the defrag machinery's unit tests exercise the -// selection/copy/re-remember mechanics directly and must bypass the -// production off-gate below. Thread-local so parallel tests don't race. -#[cfg(test)] -thread_local! { - pub(crate) static OLD_DEFRAG_TEST_OVERRIDE: std::cell::Cell> = - const { std::cell::Cell::new(None) }; -} - -/// RAII enable for the defrag unit tests: forces the off-gate open on this -/// thread for the guard's lifetime. -#[cfg(test)] -pub(crate) struct OldDefragTestEnable; - -#[cfg(test)] -impl OldDefragTestEnable { - pub(crate) fn new() -> Self { - OLD_DEFRAG_TEST_OVERRIDE.with(|c| c.set(Some(true))); - OldDefragTestEnable - } -} - -#[cfg(test)] -impl Drop for OldDefragTestEnable { - fn drop(&mut self) { - OLD_DEFRAG_TEST_OVERRIDE.with(|c| c.set(None)); - } -} - -fn old_page_defrag_enabled() -> bool { - #[cfg(test)] - if let Some(v) = OLD_DEFRAG_TEST_OVERRIDE.with(|c| c.get()) { - return v; - } - use std::sync::OnceLock; - static OPT_IN: OnceLock = OnceLock::new(); - *OPT_IN.get_or_init(|| { - matches!( - std::env::var("PERRY_GC_OLD_DEFRAG").as_deref(), - Ok("1") | Ok("on") | Ok("true") - ) - }) -} - -pub(super) fn select_old_page_defrag_pages(force: bool) -> OldPageDefragSelection { - // gh #6206: old-page defrag evacuation is OFF pending a rewrite-contract - // fix. With defrag active, a reader can observe a pre-move address of a - // defrag-moved old object long after the cycle (wild-pointer crash / - // silently corrupt cached value); the reproducer corrupts 6/6 with defrag - // enabled and is clean 6/6 with it disabled, on the same binary, while - // every heap-payload slot (arrays in-length, object fields, Map entries) - // verifies as correctly rewritten — the stale reference lives on a - // non-heap path (address-keyed cache / IC / side table) the defrag - // rewrite doesn't reach. Nursery evacuation and tenured promotion (the - // reclaim-critical moving paths) are unaffected. Re-enable for - // debugging/bisection with PERRY_GC_OLD_DEFRAG=1. - if !old_page_defrag_enabled() { - return OldPageDefragSelection::default(); - } - let snapshot = crate::arena::old_page_meta_snapshot(); - select_old_page_defrag_pages_from_snapshot(&snapshot, force) -} - pub(super) fn evacuation_policy_initial_decision( tenured_still_in_nursery_bytes: usize, rss_bytes: u64, @@ -1128,6 +989,16 @@ fn legacy_sweep_with_age_bump_and_old_reclaim_targets( } else { crate::arena::ArenaResetStats::default() }; + // #7437: rebuild the old-gen hole free list from the surviving blocks. + // Runs AFTER the block reclaim above, so a fully-dead block's holes are + // never recorded — its bytes were recycled wholesale, which is strictly + // better than hole-by-hole reuse. + if reclaim_dead_old_blocks { + old_free_rebuild_from_live_old_blocks(&block_has_live, old_block_start); + if std::env::var_os("PERRY_GC_DIAG").is_some() { + eprintln!("[gc-old-free] reusable_bytes={}", old_free_bytes()); + } + } let reset = crate::arena::ArenaResetStats { reset_blocks: nursery_reset .reset_blocks @@ -1284,6 +1155,7 @@ impl IncrementalSweepState { SweepCycleSubphase::ArenaObjects => { if self.arena.step(budget) { self.arena.maybe_print_diag(); + self.arena.push_live_block_holes(); self.cleanup = Some(ArenaSweepCleanupState::new( self.arena.block_has_live(), self.arena.block_snapshots(), @@ -1405,6 +1277,22 @@ impl ArenaSweepObjectsState { } } + /// #7437: rebuild the old-gen hole free list once the object walk + /// completes — block liveness is final at that point, and the block + /// cleanup that follows only touches blocks with NO live object, which + /// the rebuild's filter already skips. + fn push_live_block_holes(&mut self) { + if self.reclaim_dead_old_blocks { + super::old_free_rebuild_from_live_old_blocks( + &self.block_has_live, + self.old_block_start, + ); + if std::env::var_os("PERRY_GC_DIAG").is_some() { + eprintln!("[gc-old-free] reusable_bytes={}", super::old_free_bytes()); + } + } + } + fn step(&mut self, budget: usize) -> bool { let mut remaining = budget; while remaining > 0 { diff --git a/crates/perry-runtime/src/gc/oldgen_defrag.rs b/crates/perry-runtime/src/gc/oldgen_defrag.rs new file mode 100644 index 0000000000..169ce48fb2 --- /dev/null +++ b/crates/perry-runtime/src/gc/oldgen_defrag.rs @@ -0,0 +1,143 @@ +//! Old-page defragmentation SELECTION: which old-gen pages are worth +//! evacuating, and the test/env gates around that choice. Split from +//! `oldgen.rs` (2000-line lint cap); the evacuation-policy decisions that +//! CONSUME a selection stay there. + +#[derive(Default)] +pub(super) struct OldPageDefragSelection { + pub(super) pages: crate::fast_hash::PtrHashSet, + pub(super) page_order: Vec, + pub(super) candidate_pages: usize, + pub(super) selected_pages: usize, + pub(super) selected_live_bytes: usize, + pub(super) selected_reclaimable_bytes: usize, + /// Page-granule bytes the selected pages would hand back once their + /// movable live objects are evacuated: page size minus pinned bytes + /// (selection skips pinned pages, so in practice the full granule). + pub(super) selected_releasable_block_bytes: usize, + pub(super) skipped_pinned_pages: usize, +} + +#[inline] +pub(super) fn old_page_defrag_eligible(meta: crate::arena::OldPageMeta) -> bool { + meta.allocated_bytes > 0 && meta.live_bytes > 0 && meta.dead_bytes > 0 && meta.pinned_bytes == 0 +} + +#[inline] +pub(super) fn old_page_defrag_skipped_for_pin(meta: crate::arena::OldPageMeta) -> bool { + meta.allocated_bytes > 0 && meta.live_bytes > 0 && meta.dead_bytes > 0 && meta.pinned_bytes > 0 +} + +pub(super) fn select_old_page_defrag_pages_from_snapshot( + snapshot: &[crate::arena::OldPageMeta], + force: bool, +) -> OldPageDefragSelection { + let mut selection = OldPageDefragSelection::default(); + let mut candidates = Vec::new(); + for &meta in snapshot { + if old_page_defrag_skipped_for_pin(meta) { + selection.skipped_pinned_pages = selection.skipped_pinned_pages.saturating_add(1); + continue; + } + if !old_page_defrag_eligible(meta) { + continue; + } + selection.candidate_pages = selection.candidate_pages.saturating_add(1); + if force || meta.dead_bytes >= meta.live_bytes { + candidates.push(meta); + } + } + + candidates.sort_unstable_by(|a, b| { + let b_ratio = (b.dead_bytes as u128).saturating_mul(a.allocated_bytes as u128); + let a_ratio = (a.dead_bytes as u128).saturating_mul(b.allocated_bytes as u128); + b_ratio + .cmp(&a_ratio) + .then_with(|| a.live_bytes.cmp(&b.live_bytes)) + .then_with(|| a.page_base.cmp(&b.page_base)) + }); + + for meta in candidates { + let page = crate::arena::generation_page_for_addr(meta.page_base); + if selection.pages.insert(page) { + selection.page_order.push(page); + selection.selected_pages = selection.selected_pages.saturating_add(1); + selection.selected_live_bytes = selection + .selected_live_bytes + .saturating_add(meta.live_bytes); + selection.selected_reclaimable_bytes = selection + .selected_reclaimable_bytes + .saturating_add(meta.dead_bytes); + selection.selected_releasable_block_bytes = + selection.selected_releasable_block_bytes.saturating_add( + (meta.page_end.saturating_sub(meta.page_base)) + .saturating_sub(meta.pinned_bytes), + ); + } + } + + selection +} + +// gh #6206 test hook: the defrag machinery's unit tests exercise the +// selection/copy/re-remember mechanics directly and must bypass the +// production off-gate below. Thread-local so parallel tests don't race. +#[cfg(test)] +thread_local! { + pub(crate) static OLD_DEFRAG_TEST_OVERRIDE: std::cell::Cell> = + const { std::cell::Cell::new(None) }; +} + +/// RAII enable for the defrag unit tests: forces the off-gate open on this +/// thread for the guard's lifetime. +#[cfg(test)] +pub(crate) struct OldDefragTestEnable; + +#[cfg(test)] +impl OldDefragTestEnable { + pub(crate) fn new() -> Self { + OLD_DEFRAG_TEST_OVERRIDE.with(|c| c.set(Some(true))); + OldDefragTestEnable + } +} + +#[cfg(test)] +impl Drop for OldDefragTestEnable { + fn drop(&mut self) { + OLD_DEFRAG_TEST_OVERRIDE.with(|c| c.set(None)); + } +} + +fn old_page_defrag_enabled() -> bool { + #[cfg(test)] + if let Some(v) = OLD_DEFRAG_TEST_OVERRIDE.with(|c| c.get()) { + return v; + } + use std::sync::OnceLock; + static OPT_IN: OnceLock = OnceLock::new(); + *OPT_IN.get_or_init(|| { + matches!( + std::env::var("PERRY_GC_OLD_DEFRAG").as_deref(), + Ok("1") | Ok("on") | Ok("true") + ) + }) +} + +pub(super) fn select_old_page_defrag_pages(force: bool) -> OldPageDefragSelection { + // gh #6206: old-page defrag evacuation is OFF pending a rewrite-contract + // fix. With defrag active, a reader can observe a pre-move address of a + // defrag-moved old object long after the cycle (wild-pointer crash / + // silently corrupt cached value); the reproducer corrupts 6/6 with defrag + // enabled and is clean 6/6 with it disabled, on the same binary, while + // every heap-payload slot (arrays in-length, object fields, Map entries) + // verifies as correctly rewritten — the stale reference lives on a + // non-heap path (address-keyed cache / IC / side table) the defrag + // rewrite doesn't reach. Nursery evacuation and tenured promotion (the + // reclaim-critical moving paths) are unaffected. Re-enable for + // debugging/bisection with PERRY_GC_OLD_DEFRAG=1. + if !old_page_defrag_enabled() { + return OldPageDefragSelection::default(); + } + let snapshot = crate::arena::old_page_meta_snapshot(); + select_old_page_defrag_pages_from_snapshot(&snapshot, force) +} diff --git a/crates/perry-runtime/src/gc/policy.rs b/crates/perry-runtime/src/gc/policy.rs index bf509f7c9e..77a5bb494e 100644 --- a/crates/perry-runtime/src/gc/policy.rs +++ b/crates/perry-runtime/src/gc/policy.rs @@ -1155,6 +1155,16 @@ pub fn gc_schedule_parse_boundary_collection_if_pressure() { GC_SUPPRESSED_TINY_PARSE_COLLECTION_PENDING.with(|pending| pending.set(true)); } +/// Old-gen pressure the reclaim arms act on: block-offset in-use minus the +/// swept holes the free list can already hand back (#7437). Before hole +/// reuse existed, dead-but-unreclaimable bytes counted as pressure, so +/// old-reclaim kept re-firing full collections that could not actually +/// lower the number they were watching (probe 12: 49/50 blocks pinned by +/// scattered survivors, in-use immovable at ~105 MB). +pub(super) fn old_gen_reclaimable_pressure_bytes() -> usize { + crate::arena::old_gen_in_use_bytes().saturating_sub(super::old_free_bytes()) +} + #[inline] pub(super) fn old_reclaim_pressure_due(old_in_use: usize, baseline: usize) -> bool { (old_in_use >= gc_old_gen_reclaim_threshold_dyn_bytes() @@ -1218,7 +1228,7 @@ pub(super) fn copied_minor_promotion_handoff_due(trigger_kind: GcTriggerKind) -> } let promotable = copied_minor_promotable_active_survivor_bytes(); let old_in_use = - crate::arena::old_gen_in_use_bytes().saturating_add(external_side_live_bytes()); + old_gen_reclaimable_pressure_bytes().saturating_add(external_side_live_bytes()); let baseline = GC_LAST_OLD_RECLAIM_IN_USE_BYTES.with(|bytes| bytes.get()); copied_minor_promotion_handoff_pressure_due(promotable, old_in_use, baseline) } @@ -1229,7 +1239,7 @@ pub(super) fn maybe_schedule_old_reclaim_after_copied_minor() { // reclaim's old-gen sweep finalizes it, so the buffer bytes must be // able to escalate that reclaim. let old_in_use = - crate::arena::old_gen_in_use_bytes().saturating_add(external_side_live_bytes()); + old_gen_reclaimable_pressure_bytes().saturating_add(external_side_live_bytes()); let baseline = GC_LAST_OLD_RECLAIM_IN_USE_BYTES.with(|bytes| bytes.get()); if old_reclaim_pressure_due(old_in_use, baseline) { GC_OLD_RECLAIM_PENDING.with(|pending| pending.set(true)); @@ -1240,7 +1250,7 @@ pub(super) fn finish_full_old_reclaim_baseline() { // Baseline includes external side-buffer bytes (#6010) so the growth // delta in `old_reclaim_pressure_due` stays unit-consistent. let old_in_use = - crate::arena::old_gen_in_use_bytes().saturating_add(external_side_live_bytes()); + old_gen_reclaimable_pressure_bytes().saturating_add(external_side_live_bytes()); GC_LAST_OLD_RECLAIM_IN_USE_BYTES.with(|bytes| bytes.set(old_in_use)); // Record the TOTAL post-full live set for major-GC pacing (young+old): the // full sweep is the only collection that frees forwarding stubs, so this is @@ -1869,7 +1879,7 @@ fn gc_budgeted_due_trigger() -> Option { let old_pending = GC_OLD_RECLAIM_PENDING.with(Cell::get); // #6010: external Map/Set side-buffer bytes escalate to OldReclaim too. let old_in_use = - crate::arena::old_gen_in_use_bytes().saturating_add(external_side_live_bytes()); + old_gen_reclaimable_pressure_bytes().saturating_add(external_side_live_bytes()); let old_baseline = GC_LAST_OLD_RECLAIM_IN_USE_BYTES.with(|bytes| bytes.get()); if old_pending || old_reclaim_pressure_due(old_in_use, old_baseline) { return Some(BudgetedGcTrigger::OldReclaim); diff --git a/crates/perry-runtime/src/gc/tests/oldgen.rs b/crates/perry-runtime/src/gc/tests/oldgen.rs index f7ac390794..e8d925d213 100644 --- a/crates/perry-runtime/src/gc/tests/oldgen.rs +++ b/crates/perry-runtime/src/gc/tests/oldgen.rs @@ -1374,3 +1374,111 @@ fn test_full_sweep_still_finalizes_unmarked_old_object() { clear_marks(); remembered_set_clear(); } + +// --------------------------------------------------------------------------- +// #7437: old-generation hole free list. +// --------------------------------------------------------------------------- + +/// Dead old objects in a block that still holds a live object become +/// exact-size reusable holes; a same-size promotion lands in one, a +/// different-size allocation does not, and the pressure/heap accounting +/// subtracts the reusable bytes. +#[test] +fn test_dead_old_holes_in_live_blocks_are_reused_by_exact_size() { + let _isolation = copying_nursery_isolation_lock(); + reset_remembered_set(); + clear_marks(); + clear_mark_seeds(); + old_free_reset_for_test(); + crate::arena::old_pages_begin_gc_cycle(); + + // One marked anchor keeps the block out of whole-block reclaim; the + // same-size dead neighbors around it become holes. + let live = crate::arena::arena_alloc_gc_old(40, 8, GC_TYPE_STRING) as usize; + let mut dead = Vec::new(); + for _ in 0..8 { + dead.push(crate::arena::arena_alloc_gc_old(40, 8, GC_TYPE_STRING) as usize); + } + let (live_header, _) = old_test_header_and_size(live); + let (_, dead_total) = old_test_header_and_size(dead[0]); + unsafe { + (*live_header).gc_flags |= GC_FLAG_MARKED; + } + + let _sweep = sweep_with_age_bump_and_old_reclaim(false, true); + + assert!( + old_free_entry_count() >= dead.len(), + "each swept dead neighbor must become a hole (got {})", + old_free_entry_count() + ); + let free_bytes = old_free_bytes(); + assert!(free_bytes >= dead.len() * dead_total); + assert_eq!( + old_gen_reclaimable_pressure_bytes(), + crate::arena::old_gen_in_use_bytes().saturating_sub(free_bytes), + "reclaim pacing must see in-use minus the reusable holes" + ); + + // Exact-size allocation reuses a hole instead of growing the bump. + let reused = crate::arena::arena_alloc_gc_old(40, 8, GC_TYPE_STRING) as usize; + assert!( + dead.contains(&reused), + "a same-size old allocation must land in a swept hole" + ); + assert_eq!(old_free_bytes(), free_bytes - dead_total); + + // A different size must NOT take a hole (exact fit keeps GcHeader::size + // in agreement with per-object promotion accounting). + let other = crate::arena::arena_alloc_gc_old(96, 8, GC_TYPE_STRING) as usize; + assert!( + !dead.contains(&other), + "a different-size allocation must not be placed in a mismatched hole" + ); + assert_eq!(old_free_bytes(), free_bytes - dead_total); + + old_free_reset_for_test(); + clear_marks(); + remembered_set_clear(); +} + +/// A hole's block can die on a LATER cycle; the block reclaim that recycles +/// its bytes must drop the block's holes so the free list never hands out a +/// pointer into recycled memory. +#[test] +fn test_old_holes_are_dropped_when_their_block_is_reclaimed() { + let _isolation = copying_nursery_isolation_lock(); + reset_remembered_set(); + clear_marks(); + clear_mark_seeds(); + old_free_reset_for_test(); + crate::arena::old_pages_begin_gc_cycle(); + + let live = crate::arena::arena_alloc_gc_old(40, 8, GC_TYPE_STRING) as usize; + let mut dead = Vec::new(); + for _ in 0..4 { + dead.push(crate::arena::arena_alloc_gc_old(40, 8, GC_TYPE_STRING) as usize); + } + let (live_header, _) = old_test_header_and_size(live); + let (_, dead_total) = old_test_header_and_size(dead[0]); + unsafe { + (*live_header).gc_flags |= GC_FLAG_MARKED; + } + let _sweep = sweep_with_age_bump_and_old_reclaim(false, true); + assert!(old_free_entry_count() >= dead.len()); + + // Second cycle: the anchor dies too, the whole block reclaims, and the + // filter must remove every hole inside it. + crate::arena::old_pages_begin_gc_cycle(); + let _sweep2 = sweep_with_age_bump_and_old_reclaim(false, true); + while let Some(ptr) = old_free_take_exact(dead_total, None) { + assert!( + !dead.contains(&ptr) && ptr != live, + "a hole inside a reclaimed block must never be handed out" + ); + } + + old_free_reset_for_test(); + clear_marks(); + remembered_set_clear(); +}