diff --git a/changelog.d/8016-json-tape-exact-spill.md b/changelog.d/8016-json-tape-exact-spill.md new file mode 100644 index 0000000000..8b77305c41 --- /dev/null +++ b/changelog.d/8016-json-tape-exact-spill.md @@ -0,0 +1,21 @@ +### perf(json): right-size tape-materialized object spill buffers (#7267) + +JSON tape materialization previously allocated every object at the two-slot +inline floor, then let the first overflowing property create a general-purpose +array with 16 slots of growth headroom. Five-field records therefore carried a +16-slot side allocation even though the tape already knew their final width. + +Both the recursive lazy materializer and the iterative deep-input materializer +now reserve an exact-width spill buffer before storing fields. The recursive +path counts only the current object's keys by hopping across nested container +links; the iterative path reuses the key count it has already collected. The +primary object deliberately remains at `INLINE_SLOT_FLOOR`: sizing its inline +allocation to every key was benchmarked in #7267 and regressed the named field +access workload. + +On `benchmarks/json_polyglot/bench_field_access.ts`, eight interleaved +`perry-dev` runs with `PERRY_NO_AUTO_OPTIMIZE=1` reduced median time from +1266.5 ms to 1138.5 ms (-10.1%) and peak RSS from 309.3 MiB to 294.3 MiB +(-4.8%), with identical checksums. A direct-parser control was unchanged +(934.5 ms vs 935.5 ms median), isolating the improvement to tape-backed object +materialization. diff --git a/crates/perry-runtime/src/array/alloc.rs b/crates/perry-runtime/src/array/alloc.rs index 13047b3124..d1bd72fc48 100644 --- a/crates/perry-runtime/src/array/alloc.rs +++ b/crates/perry-runtime/src/array/alloc.rs @@ -140,6 +140,34 @@ pub extern "C" fn js_array_alloc_with_length(capacity: u32) -> *mut ArrayHeader ptr } +/// Allocate an exact-sized holey array for runtime-owned side storage. +/// +/// Unlike [`js_array_alloc_with_length`], this does not add +/// [`MIN_ARRAY_CAPACITY`] growth headroom. Callers must know their final width; +/// the spill buffer used by JSON tape materialization does, and padding every +/// parsed object to 16 side slots would otherwise dominate the object itself. +pub(crate) fn js_array_alloc_with_length_exact(capacity: u32) -> *mut ArrayHeader { + let ptr = arena_alloc_gc( + array_byte_size(capacity as usize), + 8, + crate::gc::GC_TYPE_ARRAY, + ) as *mut ArrayHeader; + + unsafe { + (*ptr).length = capacity; + (*ptr).capacity = capacity; + let elements_ptr = (ptr as *mut u8).add(std::mem::size_of::()) as *mut u64; + for i in 0..capacity as usize { + // GC_STORE_AUDIT(POINTER_FREE): TAG_HOLE is a non-pointer sentinel for fresh array slots. + std::ptr::write(elements_ptr.add(i), crate::value::TAG_HOLE); + } + clear_array_numeric_layout(ptr); + crate::gc::layout_init_pointer_free(ptr as *mut u8); + } + + ptr +} + /// Runtime path for `Array(value)` / `new Array(value)`. /// /// A single Number argument is interpreted as an array length and must be a diff --git a/crates/perry-runtime/src/array/mod.rs b/crates/perry-runtime/src/array/mod.rs index eba92d949a..e820a19203 100644 --- a/crates/perry-runtime/src/array/mod.rs +++ b/crates/perry-runtime/src/array/mod.rs @@ -34,7 +34,9 @@ mod subclass_tests; #[cfg(test)] mod tests; -pub(crate) use self::alloc::{array_length_range_error, js_array_alloc_pointer_elements}; +pub(crate) use self::alloc::{ + array_length_range_error, js_array_alloc_pointer_elements, js_array_alloc_with_length_exact, +}; pub use self::alloc::{ js_array_alloc, js_array_alloc_literal, js_array_alloc_with_length, js_array_alloc_with_length_longlived, js_array_constructor_single, js_array_create, diff --git a/crates/perry-runtime/src/json_tape.rs b/crates/perry-runtime/src/json_tape.rs index 5ea9813249..76ebbc7d51 100644 --- a/crates/perry-runtime/src/json_tape.rs +++ b/crates/perry-runtime/src/json_tape.rs @@ -669,9 +669,12 @@ unsafe fn materialize_object( idx: &mut usize, end_idx: usize, ) -> JSValue { + let field_count = count_object_fields(source, *idx, end_idx); let obj = crate::object::js_object_alloc(0, 0); let obj_handle = scope.root_raw_mut_ptr(obj); json_tape_safepoint(JsonTapeSafepoint::MaterializeObjectRooted, obj as usize); + let obj = obj_handle.get_raw_mut_ptr::(); + crate::object::reserve_object_spill(obj as usize, field_count); while *idx < end_idx { let Some(key_entry) = source.entry(*idx) else { break; @@ -699,6 +702,32 @@ unsafe fn materialize_object( JSValue::object_ptr(obj as *mut u8) } +/// Count only this object's keys, hopping over nested values through their +/// matching-container links. The walk allocates nothing, so it is safe for a +/// lazy source whose backing pointers may be refreshed through a GC handle. +unsafe fn count_object_fields(source: &TapeSource<'_, '_>, mut idx: usize, end_idx: usize) -> u32 { + let mut count = 0u32; + while idx < end_idx { + let Some(key) = source.entry(idx) else { + break; + }; + if key.kind != KIND_KEY { + break; + } + count = count.saturating_add(1); + idx += 1; + let Some(value) = source.entry(idx) else { + break; + }; + if value.kind == KIND_OBJ_START || value.kind == KIND_ARR_START { + idx = value.link as usize + 1; + } else { + idx += 1; + } + } + count +} + unsafe fn materialize_array( source: &TapeSource<'_, '_>, scope: &crate::gc::RuntimeHandleScope, diff --git a/crates/perry-runtime/src/json_tape/iterative.rs b/crates/perry-runtime/src/json_tape/iterative.rs index afcb94e6a9..b3677aca3f 100644 --- a/crates/perry-runtime/src/json_tape/iterative.rs +++ b/crates/perry-runtime/src/json_tape/iterative.rs @@ -40,7 +40,9 @@ unsafe fn finish_frame(frame: BuildFrame) -> Option { if keys.len() != values.len() { return None; } + let field_count = u32::try_from(keys.len()).ok()?; let object = crate::object::js_object_alloc(0, 0); + crate::object::reserve_object_spill(object as usize, field_count); for (key, value) in keys.into_iter().zip(values) { crate::object::js_object_set_field_by_name( object, diff --git a/crates/perry-runtime/src/json_tape_tests.rs b/crates/perry-runtime/src/json_tape_tests.rs index 833a4c4c14..d5f335e01f 100644 --- a/crates/perry-runtime/src/json_tape_tests.rs +++ b/crates/perry-runtime/src/json_tape_tests.rs @@ -115,6 +115,47 @@ fn tape_top_level_scalars() { assert_eq!(build_tape(b"null").unwrap().entries.len(), 1); } +#[test] +fn recursive_materializer_reserves_exact_spill_per_object_depth() { + let input = br#"{"a":1,"nested":{"n0":0,"n1":1,"n2":2,"n3":3,"n4":4},"b":2}"#; + let tape = build_tape(input).expect("valid tape"); + let nested_key = crate::string::js_string_from_bytes(b"nested".as_ptr(), 6); + + crate::gc::gc_suppress(); + let value = unsafe { materialize(&tape, input) }; + let object = (value.bits() & crate::value::POINTER_MASK) as *const crate::ObjectHeader; + let nested = crate::object::js_object_get_field_by_name(object, nested_key); + let nested = (nested.bits() & crate::value::POINTER_MASK) as *const crate::ObjectHeader; + + unsafe { + assert_eq!( + (*object).field_count, + crate::object::INLINE_SLOT_FLOOR as u32, + "known width must not enlarge the primary object" + ); + let spill = + crate::object::test_spill_buffer_addr(object as usize) as *const crate::ArrayHeader; + assert!(!spill.is_null()); + assert_eq!((*spill).capacity, 3, "count only outer-object keys"); + assert_eq!((*spill).length, 3); + + assert_eq!( + (*nested).field_count, + crate::object::INLINE_SLOT_FLOOR as u32 + ); + let nested_spill = + crate::object::test_spill_buffer_addr(nested as usize) as *const crate::ArrayHeader; + assert!(!nested_spill.is_null()); + assert_eq!( + (*nested_spill).capacity, + 5, + "reserve the nested width exactly" + ); + assert_eq!((*nested_spill).length, 5); + } + crate::gc::gc_unsuppress(); +} + #[test] fn iterative_materializer_preserves_nested_objects_arrays_and_duplicate_keys() { let input = br#"{"a":[1,true,"x"],"a":{"b":2}}"#; @@ -136,6 +177,28 @@ fn iterative_materializer_preserves_nested_objects_arrays_and_duplicate_keys() { crate::json::parse_root_restore(saved_roots); } +#[test] +fn iterative_materializer_reserves_exact_spill_without_widening_object() { + let input = br#"{"f0":0,"f1":1,"f2":2,"f3":3,"f4":4}"#; + let tape = build_tape(input).expect("valid tape"); + + crate::gc::gc_suppress(); + let value = unsafe { materialize_iterative(&tape.entries, input) }.expect("materializes"); + let object = (value.bits() & crate::value::POINTER_MASK) as *const crate::ObjectHeader; + unsafe { + assert_eq!( + (*object).field_count, + crate::object::INLINE_SLOT_FLOOR as u32 + ); + let spill = + crate::object::test_spill_buffer_addr(object as usize) as *const crate::ArrayHeader; + assert!(!spill.is_null()); + assert_eq!((*spill).capacity, 5); + assert_eq!((*spill).length, 5); + } + crate::gc::gc_unsuppress(); +} + /// `TapeEntry` is 12 bytes (u32 + u8 + padding + u32). Keeping /// this compact matters for tape-size parity with parse output: /// a 1 MB JSON blob with ~20k tokens should build a ~240 KB tape, diff --git a/crates/perry-runtime/src/object/mod.rs b/crates/perry-runtime/src/object/mod.rs index 6e8c205360..6f492bdcd0 100644 --- a/crates/perry-runtime/src/object/mod.rs +++ b/crates/perry-runtime/src/object/mod.rs @@ -130,6 +130,7 @@ mod regex_proto_thunks; mod spill; pub(crate) use spill::{ learned_inline_field_count, learned_inline_fields_hot_addr, overflow_get, overflow_set, + reserve_object_spill, }; #[cfg(test)] use spill::{object_spill_enabled, spill_capable_owner, spill_get, SPILL_MAX_FIELD_INDEX}; diff --git a/crates/perry-runtime/src/object/spill.rs b/crates/perry-runtime/src/object/spill.rs index dc3570737a..03bed6666c 100644 --- a/crates/perry-runtime/src/object/spill.rs +++ b/crates/perry-runtime/src/object/spill.rs @@ -152,6 +152,56 @@ pub(crate) fn spill_set(obj_ptr: usize, field_index: usize, vbits: u64) { } } +/// Reserve exact-width overflow storage for a freshly allocated object whose +/// final key count is already known. +/// +/// Dynamic objects normally discover their width one property at a time, so +/// the first overflow store uses an array with general-purpose growth +/// headroom. JSON tapes already encode the matching container boundary and can +/// count an object's top-level keys before materializing it. Keeping the +/// primary object at [`INLINE_SLOT_FLOOR`] avoids the measured regression from +/// widening every object, while an exact spill avoids padding every record's +/// side allocation to [`crate::array::MIN_ARRAY_CAPACITY`]. +pub(crate) fn reserve_object_spill(obj_ptr: usize, field_count: u32) { + if !object_spill_enabled() + || field_count as usize > SPILL_MAX_FIELD_INDEX + || unsafe { !spill_capable_owner(obj_ptr) } + { + return; + } + + unsafe { + let obj = obj_ptr as *mut ObjectHeader; + let inline_capacity = + std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32); + if field_count <= inline_capacity { + return; + } + + let scope = crate::gc::RuntimeHandleScope::new(); + let obj_handle = scope.root_raw_mut_ptr(obj); + object_meta_ensure(obj); + + let obj = obj_handle.get_raw_mut_ptr::(); + let meta = (*obj).meta; + if (*meta).spill != 0 { + return; + } + + let spill = crate::array::js_array_alloc_with_length_exact(field_count); + let obj = obj_handle.get_raw_mut_ptr::(); + let meta = (*obj).meta; + if (*meta).spill == 0 { + (*meta).spill = spill as u64; + crate::gc::runtime_write_barrier_slot( + meta as usize, + &(*meta).spill as *const _ as usize, + spill as u64, + ); + } + } +} + #[cfg(test)] pub(crate) type SpillSafepointHook = fn(usize);