gc: old-generation hole free list — swept holes become reusable capacity (#7437)

changelog.d/7443-oldgen-hole-free-list.md

@@ -0,0 +1,5 @@
+**GC: old-generation hole free list** (#7437) — old-gen allocation was pure bump, so a swept dead old object stayed dead capacity until its *entire* block died; a block with one live object never reset. Scattered survivors therefore pinned 105.6 MB of blocks for a ~1 MB live set (the `12_large_live_set` ratchet probe: a full collection freed 87.6 MB and reclaimed nothing, 49/50 blocks live), and the dead bytes kept counting as old-gen pressure, re-firing full collections that could not lower the number they watched.
+
+Swept dead old objects in still-live blocks now become exact-size reusable holes (`gc/old_free.rs`): a size-bucketed map rebuilt at each old-reclaiming sweep's completion from a raw-headers walk over surviving old blocks (invalidated dead headers are exactly `obj_type == 0`; the walkable-gated walkers skip them without invoking the callback, so the raw walker `old_arena_walk_all_headers_filtered` is load-bearing). `arena_alloc_gc_old` (promotions, large-object births) and its defrag-aware variant consume holes through the standard birth path; old-block reset/dealloc sites filter their ranges; the old-reclaim pacing arms and `process.memoryUsage().heapUsed` subtract the reusable bytes. Exact fit keeps `GcHeader::size` in agreement with per-object promotion accounting. `PERRY_GC_DIAG=1` prints `[gc-old-free] reusable_bytes=` after each rebuild.
+
+Measured: probe 12 retained `heapUsed` after the release-phase `gc()` drops 105.6 MB → 59.9 MB (the residual is longlived-arena and young keep-window bytes, not old-gen) at ±1% peak RSS; mid-run the reusable pool cycles 2.1 → 8.4 → 45.4 MB as holes are consumed by later promotions; all five GC benchmark traces are byte-identical. Also splits old-page defrag selection into `gc/oldgen_defrag.rs` (2000-line lint cap).

crates/perry-runtime/src/arena/allocators.rs

@@ -121,6 +121,23 @@ pub fn arena_alloc_gc_old(size: usize, align: usize, obj_type: u8) -> *mut u8 {
     // Same alignment-preservation rationale as `arena_alloc_gc`.
     let pad = align.max(8);
     let total = (GC_HEADER_SIZE + size + pad - 1) & !(pad - 1);
+    // #7437: reuse a swept same-size hole before bumping — otherwise a
+    // block with any live object never yields its dead bytes back and old
+    // capacity only ever grows. Exact fit keeps `GcHeader::size` equal to
+    // what per-object promotion accounting records for this allocation.
+    if let Some(user_ptr) = crate::gc::old_free_take_exact(total, None) {
+        let raw = (user_ptr - GC_HEADER_SIZE) as *mut u8;
+        unsafe {
+            let header = raw as *mut GcHeader;
+            (*header).obj_type = obj_type;
+            (*header).gc_flags = GC_FLAG_ARENA | crate::gc::gc_birth_extra_flags();
+            crate::gc::gc_note_black_birth(header);
+            (*header)._reserved = 0;
+            (*header).size = total as u32;
+        }
+        register_old_object_pages(raw as usize, total);
+        return user_ptr as *mut u8;
+    }
     let raw = arena_alloc_old(total, align);
 
     unsafe {
@@ -146,6 +163,21 @@ pub(crate) fn arena_alloc_gc_old_excluding_pages(
 
     let pad = align.max(8);
     let total = (GC_HEADER_SIZE + size + pad - 1) & !(pad - 1);
+    // #7437: same hole reuse as `arena_alloc_gc_old`, but never into a page
+    // this defrag pass is evacuating.
+    if let Some(user_ptr) = crate::gc::old_free_take_exact(total, Some(excluded_pages)) {
+        let raw = (user_ptr - GC_HEADER_SIZE) as *mut u8;
+        unsafe {
+            let header = raw as *mut GcHeader;
+            (*header).obj_type = obj_type;
+            (*header).gc_flags = GC_FLAG_ARENA | crate::gc::gc_birth_extra_flags();
+            crate::gc::gc_note_black_birth(header);
+            (*header)._reserved = 0;
+            (*header).size = total as u32;
+        }
+        register_old_object_pages(raw as usize, total);
+        return user_ptr as *mut u8;
+    }
     let raw = arena_alloc_old_excluding_pages(total, align, excluded_pages);
 
     unsafe {

crates/perry-runtime/src/arena/mod.rs

@@ -70,8 +70,8 @@ pub use walk::{
 };
 pub(crate) use walk::{
     arena_block_snapshots, arena_telemetry_snapshot, general_block_in_recent_window,
-    general_block_sizes, ArenaBlockSnapshot, ArenaObjectCursor, ArenaObjectCursorBuilder,
-    ArenaTelemetrySnapshot, ArenaWalkOrder,
+    general_block_sizes, old_arena_walk_all_headers_filtered, ArenaBlockSnapshot,
+    ArenaObjectCursor, ArenaObjectCursorBuilder, ArenaTelemetrySnapshot, ArenaWalkOrder,
 };
 
 // reset.rs

crates/perry-runtime/src/arena/reset.rs

@@ -1025,6 +1025,9 @@ impl OldArenaReclaimDeadBlocksState {
             let last_page = generation_page_for_addr(base + size - 1);
             let pages: Vec<usize> = (first_page..=last_page).collect();
             unregister_old_block_pages(&pages);
+            // #7437: this block's bytes are being recycled; any swept hole
+            // recorded inside it must not be handed out again.
+            crate::gc::old_free_filter_range(base, size);
 
             if used != 0 {
                 self.stats.reset_blocks = self.stats.reset_blocks.saturating_add(1);
@@ -1115,6 +1118,9 @@ pub(crate) fn old_arena_reclaim_dead_blocks(block_has_live: &[bool]) -> ArenaRes
             let last_page = generation_page_for_addr(base + size - 1);
             let pages: Vec<usize> = (first_page..=last_page).collect();
             unregister_old_block_pages(&pages);
+            // #7437: this block's bytes are being recycled; any swept hole
+            // recorded inside it must not be handed out again.
+            crate::gc::old_free_filter_range(base, size);
 
             if used != 0 {
                 stats.reset_blocks = stats.reset_blocks.saturating_add(1);
@@ -1217,6 +1223,9 @@ pub(crate) fn old_arena_reclaim_selected_dead_blocks(
             let last_page = generation_page_for_addr(base + size - 1);
             let pages: Vec<usize> = (first_page..=last_page).collect();
             unregister_old_block_pages(&pages);
+            // #7437: this block's bytes are being recycled; any swept hole
+            // recorded inside it must not be handed out again.
+            crate::gc::old_free_filter_range(base, size);
 
             if used != 0 {
                 stats.reset_blocks = stats.reset_blocks.saturating_add(1);

crates/perry-runtime/src/arena/stats.rs

@@ -50,6 +50,13 @@ pub extern "C" fn js_arena_stats(out_used: *mut u64, out_total: *mut u64) {
             total += block.size as u64;
         }
     });
+    // #7437: swept old-gen holes are reusable capacity, not used heap.
+    // Block offsets cannot express a hole (the bump pointer never moves
+    // back), so without this subtraction `heapUsed` reports the scattered-
+    // survivor high-water forever — 105.6 MB for a ~1 MB live set on the
+    // 12_large_live_set ratchet probe — and looks like a leak that no
+    // amount of collecting can fix.
+    used = used.saturating_sub(crate::gc::old_free_bytes() as u64);
     unsafe {
         *out_used = used;
         *out_total = total;

crates/perry-runtime/src/arena/walk.rs

@@ -839,3 +839,43 @@ pub fn longlived_end() -> usize {
     let l = LONGLIVED_ARENA.with(|arena| unsafe { (*arena.get()).blocks.len() });
     g + s0 + s1 + l
 }
+
+/// #7437: walk EVERY header in the selected old-gen blocks, including
+/// invalidated dead ones (`obj_type == 0`), which the walkable-gated
+/// walkers above deliberately skip. Stepping is by `GcHeader::size`, which
+/// `invalidate_dead_old_arena_header` preserves exactly so holes remain
+/// traversable. `block_filter` receives GLOBAL block indices (same base as
+/// `arena_walk_objects_filtered`'s old-gen region).
+pub(crate) fn old_arena_walk_all_headers_filtered(
+    mut block_filter: impl FnMut(usize) -> bool,
+    mut callback: impl FnMut(*mut u8, usize),
+) {
+    use crate::gc::GcHeader;
+    let old_block_start = longlived_end();
+    OLD_ARENA.with(|arena| {
+        let arena = unsafe { &*arena.get() };
+        for (i, block) in arena.blocks.iter().enumerate() {
+            let block_idx = old_block_start + i;
+            if block.data.is_null() || !block_filter(block_idx) {
+                continue;
+            }
+            let mut offset = 0usize;
+            while offset < block.offset {
+                let aligned = (offset + 7) & !7;
+                if aligned >= block.offset {
+                    break;
+                }
+                let header_ptr = unsafe { block.data.add(aligned) };
+                let header = header_ptr as *const GcHeader;
+                unsafe {
+                    let total_size = (*header).size as usize;
+                    if total_size == 0 || total_size > block.size {
+                        break;
+                    }
+                    callback(header_ptr, block_idx);
+                    offset = aligned + total_size;
+                }
+            }
+        }
+    });
+}

crates/perry-runtime/src/gc/mod.rs

@@ -83,10 +83,15 @@ use copying::*;
 // pass in `crate::weakref` (#6182), which lives outside the gc module.
 pub(crate) use copying::CopyingPointerSet;
 mod dead_owner;
+mod old_free;
+use old_free::*;
+pub(crate) use old_free::{old_free_bytes, old_free_filter_range, old_free_take_exact};
 mod tenuring;
 use tenuring::*;
 mod oldgen;
 use oldgen::*;
+mod oldgen_defrag;
+use oldgen_defrag::*;
 mod cycle;
 use cycle::*;
 mod verify;

crates/perry-runtime/src/gc/old_free.rs

@@ -0,0 +1,193 @@
+//! Old-generation hole free list (#7437).
+//!
+//! Old-gen allocation was pure bump: a swept dead old object stayed dead
+//! capacity until its *entire block* died, and a block with even one live
+//! object never resets. A workload that promotes a large cohort and keeps
+//! a scattered subset (every-64th node in the `12_large_live_set` ratchet
+//! probe) therefore retained 105 MB of blocks for a ~1 MB live set — the
+//! final full collection freed 87 MB of objects and reclaimed nothing,
+//! because 49 of 50 blocks still held at least one live object. The same
+//! mechanism is a large slice of tree.ts's old-gen churn high-water
+//! (#7438): every dropped tree leaves holes in blocks pinned live by the
+//! next tree's nodes.
+//!
+//! This module gives the old generation what the general arena has had
+//! all along (`ARENA_FREE_LIST`): swept holes become reusable. Shape
+//! differences are deliberate:
+//!
+//! - **Exact fit only, keyed by total (header-inclusive, padded) size.**
+//!   The general list best-fits into larger slots and keeps the slot's
+//!   original `GcHeader::size`, which is fine there because nothing else
+//!   accounts those bytes. Old-gen promotion *does* account per-object
+//!   sizes (`old_page_account_promoted_object`), so a reused slot must
+//!   have exactly the size the caller asked for or the page live-byte
+//!   accounting diverges from the header. Promoted cohorts are dominated
+//!   by uniform class-instance sizes, so exact fit hits where the
+//!   pathology lives.
+//! - **Size-bucketed map, not a scanned Vec.** The pathological case has
+//!   hundreds of thousands of holes; a per-allocation linear scan would
+//!   put an O(holes) tax on every promotion.
+//!
+//! `OLD_ARENA_FREE_BYTES` tracks the total. It is deliberately NOT
+//! subtracted from `OLD_GEN_IN_USE_BYTES` — that cache is defined (and
+//! debug-asserted) as the sum of old block offsets, which hole reuse does
+//! not change. Consumers that want *live* old pressure (the old-reclaim
+//! pacing arms, `process.memoryUsage().heapUsed`) subtract
+//! [`old_free_bytes`] instead; before this, dead-but-unreclaimable bytes
+//! counted as pressure, so old-reclaim kept re-firing full collections
+//! that could not actually lower the number they were watching.
+//!
+//! Entries are only pushed for dead objects in blocks that still hold a
+//! live object (fully-dead blocks go through block reclaim, which is
+//! strictly better). A pushed entry's block can still die on a LATER
+//! cycle, so every old-block reset/dealloc site must call
+//! [`old_free_filter_range`] for the range it is about to recycle.
+
+use super::*;
+
+thread_local! {
+    /// total_size -> user_ptrs of swept holes of exactly that size.
+    static OLD_FREE_MAP: RefCell<crate::fast_hash::PtrHashMap<usize, Vec<usize>>> =
+        RefCell::new(crate::fast_hash::new_ptr_hash_map());
+    static OLD_FREE_BYTES: Cell<usize> = const { Cell::new(0) };
+    static OLD_FREE_NONEMPTY: Cell<bool> = const { Cell::new(false) };
+}
+
+/// Total bytes currently sitting in reusable old-gen holes.
+pub(crate) fn old_free_bytes() -> usize {
+    OLD_FREE_BYTES.with(Cell::get)
+}
+
+fn old_free_push(user_ptr: usize, total_size: usize) {
+    if user_ptr == 0 || total_size < GC_HEADER_SIZE {
+        return;
+    }
+    OLD_FREE_MAP.with(|m| {
+        m.borrow_mut().entry(total_size).or_default().push(user_ptr);
+    });
+    OLD_FREE_BYTES.with(|c| c.set(c.get().saturating_add(total_size)));
+    OLD_FREE_NONEMPTY.with(|c| c.set(true));
+}
+
+/// Rebuild the hole map from the heap itself: every invalidated dead
+/// header (`obj_type == 0` — only `invalidate_dead_old_arena_header`
+/// produces those; no live object has type 0) inside an old block that
+/// still holds a live object. Called at the completion point of every
+/// old-reclaiming sweep, replacing whatever the map held.
+///
+/// Rebuilding beats accumulating a staging vector during the sweep walk on
+/// two counts, both measured on `12_large_live_set` (~700k dead old
+/// objects): the staging vector alone added ~17 MB of peak RSS to the very
+/// number this feature exists to lower, and rebuild is idempotent — a hole
+/// consumed by reuse gets a real `obj_type` and drops out, a hole whose
+/// block died is never visited, so no cross-sweep dedup bookkeeping can
+/// drift. The walk is block-filtered (live old blocks only), so its cost
+/// is O(objects in surviving old blocks), paid only on reclaim sweeps.
+pub(super) fn old_free_rebuild_from_live_old_blocks(
+    block_has_live: &[bool],
+    old_block_start: usize,
+) {
+    OLD_FREE_MAP.with(|m| m.borrow_mut().clear());
+    OLD_FREE_BYTES.with(|c| c.set(0));
+    OLD_FREE_NONEMPTY.with(|c| c.set(false));
+    // The raw-headers walker is load-bearing: the walkable-gated walkers
+    // (`arena_walk_objects_filtered` and friends) step over invalidated
+    // headers WITHOUT invoking the callback, so a rebuild written against
+    // them silently records zero holes.
+    crate::arena::old_arena_walk_all_headers_filtered(
+        |block_idx| {
+            block_idx >= old_block_start && block_has_live.get(block_idx).copied().unwrap_or(false)
+        },
+        |header_ptr, _block_idx| {
+            let header = header_ptr as *mut GcHeader;
+            unsafe {
+                if (*header).obj_type == 0 {
+                    let total_size = (*header).size as usize;
+                    old_free_push(header as usize + GC_HEADER_SIZE, total_size);
+                }
+            }
+        },
+    );
+}
+
+/// Take a hole of exactly `total_size` bytes, if one exists. When
+/// `excluded_pages` is non-empty the caller is mid-defrag and must not
+/// allocate on the pages it is evacuating; holes on those pages are
+/// skipped (and retained).
+pub(crate) fn old_free_take_exact(
+    total_size: usize,
+    excluded_pages: Option<&crate::fast_hash::PtrHashSet<usize>>,
+) -> Option<usize> {
+    if !OLD_FREE_NONEMPTY.with(Cell::get) {
+        return None;
+    }
+    let taken = OLD_FREE_MAP.with(|m| {
+        let mut map = m.borrow_mut();
+        let bucket = map.get_mut(&total_size)?;
+        let taken = match excluded_pages {
+            None => bucket.pop(),
+            Some(excluded) => {
+                let idx = bucket.iter().rposition(|&ptr| {
+                    let header = ptr - GC_HEADER_SIZE;
+                    let first = crate::arena::generation_page_for_addr(header);
+                    let last = crate::arena::generation_page_for_addr(header + total_size - 1);
+                    (first..=last).all(|page| !excluded.contains(&page))
+                })?;
+                Some(bucket.swap_remove(idx))
+            }
+        };
+        if bucket.is_empty() {
+            map.remove(&total_size);
+        }
+        taken
+    })?;
+    OLD_FREE_BYTES.with(|c| c.set(c.get().saturating_sub(total_size)));
+    OLD_FREE_MAP.with(|m| {
+        if m.borrow().is_empty() {
+            OLD_FREE_NONEMPTY.with(|c| c.set(false));
+        }
+    });
+    Some(taken)
+}
+
+/// Drop every hole inside `[base, base + size)`. Called by the old-block
+/// reset/dealloc paths before they recycle a block's bytes — a stale
+/// entry would otherwise hand out a pointer into memory the bump
+/// allocator is about to overwrite (or that has been returned to the OS).
+pub(crate) fn old_free_filter_range(base: usize, size: usize) {
+    if !OLD_FREE_NONEMPTY.with(Cell::get) || size == 0 {
+        return;
+    }
+    let end = base.saturating_add(size);
+    let mut removed_bytes = 0usize;
+    OLD_FREE_MAP.with(|m| {
+        let mut map = m.borrow_mut();
+        map.retain(|&slot_size, bucket| {
+            bucket.retain(|&ptr| {
+                let header = ptr - GC_HEADER_SIZE;
+                let inside = header >= base && header < end;
+                if inside {
+                    removed_bytes = removed_bytes.saturating_add(slot_size);
+                }
+                !inside
+            });
+            !bucket.is_empty()
+        });
+        if map.is_empty() {
+            OLD_FREE_NONEMPTY.with(|c| c.set(false));
+        }
+    });
+    OLD_FREE_BYTES.with(|c| c.set(c.get().saturating_sub(removed_bytes)));
+}
+
+#[cfg(test)]
+pub(super) fn old_free_reset_for_test() {
+    OLD_FREE_MAP.with(|m| m.borrow_mut().clear());
+    OLD_FREE_BYTES.with(|c| c.set(0));
+    OLD_FREE_NONEMPTY.with(|c| c.set(false));
+}
+
+#[cfg(test)]
+pub(super) fn old_free_entry_count() -> usize {
+    OLD_FREE_MAP.with(|m| m.borrow().values().map(|b| b.len()).sum())
+}