diff --git a/benchmarks/repsel_census/fixtures/fixture_alloc_buckets.ts b/benchmarks/repsel_census/fixtures/fixture_alloc_buckets.ts index 317031e565..d5e7c74e64 100644 --- a/benchmarks/repsel_census/fixtures/fixture_alloc_buckets.ts +++ b/benchmarks/repsel_census/fixtures/fixture_alloc_buckets.ts @@ -78,15 +78,15 @@ const boxed = new Boxed(new Point(1, 2)); // with the line above. const nested = { inner: { a: 1, b: 2 }, k: 3 }; -// Bucket 4 — a returned expression OPERAND. What this function returns is the -// conditional, not either allocation, so #7107's return-shape fact covers -// neither and `pickPoint` gets no fact at all (`producer_return_class` admits -// only a bare `Expr::New` or a proven local as a return). Before the #7176 -// review both arms inherited the `return` label from `Stmt::Return` and were -// counted as return positions — which is what over-stated the `return` bucket -// published on #7170 as R1's ceiling. +// Buckets 4 and 5 — returned expression OPERANDS on opposite sides of the R2 +// boundary. The allocation in the conditional's condition is evaluated but +// can never become the returned value, so it remains an unserved rule-1 wall. +// The two fresh, agreeing result arms do feed the return-shape fact and must be +// reported as served. All three retain the operand position rather than being +// mislabeled as direct returns, which is the #7176 distinction this fixture +// originally introduced. export function pickPoint(flag: boolean): Point { - return flag ? new Point(5, 6) : new Point(7, 8); + return (flag && new Point(0, 0)) ? new Point(5, 6) : new Point(7, 8); } const p = makePoint(3); diff --git a/changelog.d/8086-authoritative-shape-guards.md b/changelog.d/8086-authoritative-shape-guards.md new file mode 100644 index 0000000000..8e6ca0cf98 --- /dev/null +++ b/changelog.d/8086-authoritative-shape-guards.md @@ -0,0 +1,8 @@ +### Fixed + +- Make `ShapeId` descriptors authoritative for runtime and generated object + guards, including moving keys, exact logical/live slot facts, semantic + transitions, agent-local installation, and fail-stop exhaustion. Class + objects now carry their kind in the authoritative descriptor, and RegExp + values use a dedicated GC kind with relocation-safe side tables, while the + object header size remains unchanged. diff --git a/crates/perry-codegen/src/collectors/proven_this.rs b/crates/perry-codegen/src/collectors/proven_this.rs index d12e670fa6..4d40ec03ba 100644 --- a/crates/perry-codegen/src/collectors/proven_this.rs +++ b/crates/perry-codegen/src/collectors/proven_this.rs @@ -68,14 +68,11 @@ //! //! * `method_direct.fast` (`lower_call/method_override.rs`) sits behind //! `js_method_direct_shape_guard` / `js_typed_feedback_method_direct_call_guard`, -//! whose contract includes `receiver.keys_array == expected_keys` — a raw -//! POINTER compare (`typed_feedback/guards.rs`). The only code path -//! `js_object_delete_field` has for a `GC_TYPE_OBJECT` instance with a -//! keys array clones a FRESH keys array and repoints `keys_array` at it -//! (`perry-runtime/src/object/delete_rest.rs`; `Reflect.deleteProperty` -//! shares the same function) — for ANY key, declared or not, from ANY -//! module. The pointer compare can therefore never pass on a post-delete -//! instance, regardless of what this admission check saw. +//! whose contract includes `receiver.ShapeId == expected_shape_id` +//! (`typed_feedback/guards.rs`). `js_object_delete_field` publishes a +//! semantic successor ShapeId (`perry-runtime/src/object/delete_rest.rs`; +//! `Reflect.deleteProperty` shares the same function), so the guard can +//! never pass on a post-delete instance regardless of module boundaries. //! * The Phase 3b guard-free `Ptr` receiver arm needs no runtime //! check at all, because rule 2's containment already rules out the alias //! existing in the first place: creating one — `let other = o`, passing diff --git a/crates/perry-codegen/src/collectors/proven_this_routing_tests.rs b/crates/perry-codegen/src/collectors/proven_this_routing_tests.rs index 99b386d710..9f3d7cfe8d 100644 --- a/crates/perry-codegen/src/collectors/proven_this_routing_tests.rs +++ b/crates/perry-codegen/src/collectors/proven_this_routing_tests.rs @@ -23,7 +23,7 @@ //! Both routing sites are covered: //! //! * `lower_call/method_override.rs` — the guarded `method_direct.fast` arm, -//! dominated by the class-id + keys-token guard. +//! dominated by the class-id + ShapeId guard. //! * `lower_call/property_get/dynamic_dispatch.rs` — the Phase 3b guard-free //! `Ptr` receiver arm. //! @@ -403,7 +403,7 @@ fn ptr_shape_local_module() -> Module { /// A module whose `probe(c: Counter)` calls both methods on a statically-typed /// parameter. A typed parameter is not a Phase 3b shape-proven local (no /// provenance, no containment), so both calls go through the *guarded* site: -/// `emit_guarded_direct_method_call`, behind the class-id + keys-token guard. +/// `emit_guarded_direct_method_call`, behind the class-id + ShapeId guard. fn guarded_site_module() -> Module { let mut m = Module::new("pshape_guarded.ts"); m.classes = vec![counter_class()]; @@ -495,7 +495,7 @@ fn function_body(ir: &str, name_contains: &str) -> String { } /// Soundness ratchet (#7143): `method_direct.fast`'s `$pshape` call site is -/// preceded, in its own function, by the keys-token shape guard. +/// preceded, in its own function, by the ShapeId guard. /// /// `guarded_site_module`'s `probe(c: Counter)` receiver (`c`) is a plain /// typed PARAMETER — proven-`this`'s aliased-by-construction case, not a @@ -511,25 +511,24 @@ fn function_body(ir: &str, name_contains: &str) -> String { /// (`lower_call/method_override.rs`) unconditionally emits /// `js_typed_feedback_method_direct_call_guard` (or, under `shape_only_guard`, /// `js_method_direct_shape_guard`) BEFORE any block that can reach the clone -/// — both compare the receiver's live `keys_array` pointer against the -/// class's canonical `@perry_class_keys_*` token, and `delete`'s only code -/// path for a class instance always repoints `keys_array` at a freshly -/// cloned array (`perry-runtime/src/object/delete_rest.rs`), from ANY +/// — both compare the receiver's live ShapeId against the class's canonical +/// `@perry_class_shape_id_*` value, and `delete` publishes a semantic +/// successor descriptor (`perry-runtime/src/object/delete_rest.rs`), from ANY /// module. See `collectors/proven_this.rs`'s "`delete` is aliased across /// modules by construction" section for the full argument; this pins the /// IR shape it depends on, the same way -/// `tower_route_is_guarded_by_the_class_keys_token` pins it for the #7142 +/// `tower_route_is_guarded_by_the_class_shape_id` pins it for the #7142 /// tower site. /// /// This checks TEXTUAL precedence within `probe`'s body rather than walking -/// block dominance (`tower_route_is_guarded_by_the_class_keys_token`'s +/// block dominance (`tower_route_is_guarded_by_the_class_shape_id`'s /// approach) because `probe` calls two methods sequentially and the typed-f64 /// arm nests the clone's call another level deep behind its OWN per-argument /// guard — precedence is the invariant that survives that nesting, and /// `probe` is this fixture's only method-dispatching function, so it is also /// the only place a `$pshape` callee name can appear in a `call` line. #[test] -fn guarded_pshape_call_site_is_preceded_by_a_keys_token_guard() { +fn guarded_pshape_call_site_is_preceded_by_a_shape_id_guard() { let ir = emit(&guarded_site_module(), false); let calls = pshape_call_targets(&ir); assert!( @@ -546,7 +545,7 @@ fn guarded_pshape_call_site_is_preceded_by_a_keys_token_guard() { || prefix.contains("call i32 @js_method_direct_shape_guard("); assert!( guarded, - "{target}: no keys-token guard call precedes it in `probe` — a \ + "{target}: no ShapeId guard call precedes it in `probe` — a \ post-`delete` receiver (deleted from through an alias in another \ module, #7143) would reach this clone's stale fixed-slot loads \ unguarded:\n{probe}" @@ -770,16 +769,16 @@ fn tower_case_routes_to_proven_this_clone() { } /// Soundness ratchet (#7142): the routed call is dominated by a compare of the -/// receiver's live `keys_array` against the class's `@perry_class_keys_*` token. +/// receiver's authoritative ShapeId against `@perry_class_shape_id_*`. /// /// A `class_id` match alone is NOT a layout proof — `delete inst.f` compacts /// the packed slots while preserving `class_id` /// (`object/delete_rest.rs`), which is why #7141 refused to route this site at /// all. The compare below is the entire difference between sound and unsound, -/// so it is traced end to end: global → entry-hoisted slot → reload in the -/// guard block → `icmp eq i64` → the branch that enters the clone's block. +/// so it is traced end to end: global → entry-hoisted scalar slot → reload in +/// the guard block → `icmp eq i32` → the branch that enters the clone's block. #[test] -fn tower_route_is_guarded_by_the_class_keys_token() { +fn tower_route_is_guarded_by_the_class_shape_id() { let _shadow = crate::codegen::helpers::NativeRootsPin::shadow(); let ir = emit(&tower_site_module(), false); let bs = blocks(&ir); @@ -801,51 +800,40 @@ fn tower_route_is_guarded_by_the_class_keys_token() { panic!("nothing conditionally branches to {clone_block} — the clone is reached unguarded:\n{ir}") }); - // 1. the class keys global is loaded once at function entry … + // 1. the class ShapeId global is loaded once at function entry … let global_load = ir .lines() - .find(|l| l.contains("= load i64, ptr @perry_class_keys_")) - .unwrap_or_else(|| panic!("the class keys token is never read:\n{ir}")); + .find(|l| l.contains("= load i32, ptr @perry_class_shape_id_")) + .unwrap_or_else(|| panic!("the class ShapeId is never read:\n{ir}")); let global_reg = global_load.trim().split(' ').next().expect("ssa name"); // 2. … and parked in an entry slot … let store = ir .lines() - .find(|l| l.contains(&format!("store i64 {}, ptr ", global_reg))) - .unwrap_or_else(|| panic!("the hoisted keys token is never stored:\n{ir}")); + .find(|l| l.contains(&format!("store i32 {}, ptr ", global_reg))) + .unwrap_or_else(|| panic!("the hoisted ShapeId is never stored:\n{ir}")); let slot = store.rsplit(' ').next().expect("slot name"); - let store_pos = ir - .find(store) - .expect("the hoisted class-keys store should be in the function"); - let bind_pos = ir - .lines() - .find(|line| line.contains("call void @js_shadow_slot_bind") && line.contains(slot)) - .and_then(|line| ir.find(line)) - .unwrap_or_else(|| { - panic!( - "the cached class-keys pointer is not a mutable shadow root; old-page moves would leave this copy stale:\n{ir}" - ) - }); assert!( - store_pos < bind_pos, - "the class-keys slot must be initialized before the root scanner can read it:\n{ir}" + !ir.lines() + .any(|line| line.contains("call void @js_shadow_slot_bind") && line.contains(slot)), + "a ShapeId scalar must not be registered as a moving GC root:\n{ir}" ); // 3. … which the guard block reloads … let expected = guard_body .iter() .find_map(|l| { let l = l.trim(); - l.ends_with(&format!("load i64, ptr {}", slot)) + l.ends_with(&format!("load i32, ptr {}", slot)) .then(|| l.split(' ').next().expect("ssa name").to_string()) }) .unwrap_or_else(|| { - panic!("the guard block never reads the hoisted keys token:\n{guard_body:#?}") + panic!("the guard block never reads the hoisted ShapeId:\n{guard_body:#?}") }); - // 4. … and compares against the receiver's live keys_array. + // 4. … and compares against the receiver's live ShapeId. assert!( guard_body .iter() - .any(|l| l.contains("icmp eq i64") && l.contains(&expected)), - "the routed call is not dominated by a keys-token compare — a class_id \ + .any(|l| l.contains("icmp eq i32") && l.contains(&expected)), + "the routed call is not dominated by a ShapeId compare — a class_id \ match alone does not prove the packed layout (`delete` compacts slots \ while preserving class_id):\n{guard_body:#?}" ); @@ -861,30 +849,18 @@ fn tower_route_is_guarded_by_the_class_keys_token() { } #[test] -fn tower_class_keys_cache_is_a_native_mutable_root() { +fn tower_class_shape_id_cache_is_a_native_scalar() { let _native = crate::codegen::helpers::NativeRootsPin::native(); let ir = emit(&tower_site_module(), false); let global_load = ir .lines() - .find(|line| line.contains("= load i64, ptr @perry_class_keys_")) - .unwrap_or_else(|| panic!("the class keys token is never read:\n{ir}")); + .find(|line| line.contains("= load i32, ptr @perry_class_shape_id_")) + .unwrap_or_else(|| panic!("the class ShapeId is never read:\n{ir}")); let global_reg = global_load.trim().split(' ').next().expect("ssa name"); - let cast = ir - .lines() - .find(|line| line.contains(&format!("= inttoptr i64 {global_reg} to ptr addrspace(1)"))) - .unwrap_or_else(|| { - panic!("the cached class-keys pointer never enters a native GC root slot:\n{ir}") - }); - let cast_reg = cast.trim().split(' ').next().expect("cast ssa name"); - let root_store = ir - .lines() - .find(|line| line.contains(&format!("store ptr addrspace(1) {cast_reg}, ptr "))) - .unwrap_or_else(|| panic!("the native class-keys root is never stored:\n{ir}")); - let slot = root_store.rsplit(' ').next().expect("root slot name"); assert!( ir.lines() - .any(|line| line.contains(&format!("{slot} = alloca ptr addrspace(1)"))), - "the class-keys cache alloca must be in the collector address space:\n{ir}" + .any(|line| line.contains(&format!("store i32 {global_reg}, ptr "))), + "the class ShapeId must be cached as an i32 scalar:\n{ir}" ); } diff --git a/crates/perry-codegen/src/expr/class_field_inline_guard.rs b/crates/perry-codegen/src/expr/class_field_inline_guard.rs index fd7e8c49a0..e3499005e9 100644 --- a/crates/perry-codegen/src/expr/class_field_inline_guard.rs +++ b/crates/perry-codegen/src/expr/class_field_inline_guard.rs @@ -45,7 +45,6 @@ const POINTER_TAG_HI16: &str = "32765"; // 0x7FFD — NaN-box tag for heap point const HANDLE_BAND_TOP: &str = "1048575"; // 0x0FFFFF — handles are <= this; objects are above const GC_TYPE_OBJECT: &str = "2"; const GC_FLAG_FORWARDED_I8: &str = "-128"; // 0x80 as i8 -const OBJECT_TYPE_REGULAR: &str = "1"; const TYPED_LAYOUT_INTACT_BIT: &str = "4096"; // GC_OBJ_TYPED_LAYOUT_INTACT (0x1000) const OBJ_FLAG_FROZEN_BIT: &str = "1"; // OBJ_FLAG_FROZEN (0x01) const OBJ_FLAG_HAS_DESCRIPTORS_BIT: &str = "2048"; // OBJ_FLAG_HAS_DESCRIPTORS (0x800) @@ -62,7 +61,7 @@ const F64_EXP_MASK: &str = "9218868437227405312"; // 0x7FF0_0000_0000_0000 #[derive(Clone, Debug)] pub(crate) struct ClassFieldSubclassArm { pub class_id: u32, - pub keys_global: String, + pub shape_id_global: String, } /// A hierarchy wider than this turns the shape check into a longer compare @@ -153,7 +152,9 @@ pub(crate) fn class_field_subclass_arms( seen_ids.push(sub_id); arms.push(ClassFieldSubclassArm { class_id: sub_id, - keys_global, + shape_id_global: crate::typed_shape::shape_id_global_name_from_keys_global( + &keys_global, + ), }); if arms.len() > MAX_CLASS_FIELD_SUBCLASS_ARMS { return Vec::new(); @@ -209,9 +210,7 @@ pub(crate) fn emit_plain_finite_number_check( /// keys_array / field_count / the typed-layout intact bit / the frozen bit / /// the process-global enable flag mid-loop. /// -/// `max_field_index` is the largest packed slot index the loop touches -/// (`field_count ugt max_field_index` covers every access). `require_raw_f64` -/// adds the per-object typed-layout intact check (any raw-f64 read or write in +/// `require_raw_f64` adds the per-object typed-layout intact check (any raw-f64 read or write in /// the loop); `require_not_frozen` adds the frozen-bit check (any write in the /// loop). Per-store value checks are NOT emitted here — the fast clone's /// stores keep their inline plain-finite check and side-exit to `slow_label`. @@ -233,15 +232,13 @@ pub(crate) fn emit_class_field_loop_preheader_check( obj_bits: &str, obj_handle: &str, expected_class_id: &str, - expected_keys: &str, - max_field_index: u32, + expected_shape_id: &str, require_raw_f64: bool, require_not_frozen: bool, slow_label: &str, ) -> (String, String) { let deref_idx = ctx.new_block("class_field_loop.preheader.deref"); let deref_label = ctx.block_label(deref_idx); - let max_field_index_str = max_field_index.to_string(); // Gate: enable flag first (volatile — the runtime flips it sticky 0 -> 1 // when descriptors / typed feedback / verify mode come into use), then @@ -277,28 +274,19 @@ pub(crate) fn emit_class_field_loop_preheader_check( let res_ptr = blk.gep(I8, &obj_ptr, &[(I64, "-6")]); let reserved = blk.load(I16, &res_ptr); - // ObjectHeader: object_type @0 (i32)==REGULAR, class_id @4 (i32), - // field_count @12 (i32), keys_array @16 (i64). - let object_type = blk.load(I32, &obj_ptr); - let ot_ok = blk.icmp_eq(I32, &object_type, OBJECT_TYPE_REGULAR); - + // ObjectHeader: class_id @4 and authoritative ShapeId @8. Matching + // the immutable descriptor proves the live-slot bound and key order. let cid_ptr = blk.gep(I8, &obj_ptr, &[(I64, "4")]); let class_id = blk.load(I32, &cid_ptr); let cid_ok = blk.icmp_eq(I32, &class_id, expected_class_id); - let fc_ptr = blk.gep(I8, &obj_ptr, &[(I64, "12")]); - let field_count = blk.load(I32, &fc_ptr); - let fc_ok = blk.icmp_ugt(I32, &field_count, &max_field_index_str); - - let ka_ptr = blk.gep(I8, &obj_ptr, &[(I64, "16")]); - let keys_array = blk.load(I64, &ka_ptr); - let ka_ok = blk.icmp_eq(I64, &keys_array, expected_keys); + let sid_ptr = blk.gep(I8, &obj_ptr, &[(I64, "8")]); + let shape_id = blk.load(I32, &sid_ptr); + let shape_ok = blk.icmp_eq(I32, &shape_id, expected_shape_id); let mut acc = blk.and(I1, >ype_ok, ¬_fwd); - acc = blk.and(I1, &acc, &ot_ok); acc = blk.and(I1, &acc, &cid_ok); - acc = blk.and(I1, &acc, &fc_ok); - acc = blk.and(I1, &acc, &ka_ok); + acc = blk.and(I1, &acc, &shape_ok); // #5654: a receiver that has ever had a property / accessor descriptor // installed on it needs the guard's descriptor-aware dispatch (an @@ -345,14 +333,12 @@ pub(crate) fn emit_class_field_loop_preheader_check( /// /// ## What is left, and why each one /// -/// * **`keys_array` identity** — the load-bearing one. `delete inst.f` compacts +/// * **ShapeId identity** — the load-bearing one. `delete inst.f` compacts /// the packed inline slots while PRESERVING `class_id`, so a class-id match /// alone does not prove the layout: on `class C { a; b; c }`, `delete inst.b` -/// moves `c` from slot 2 to slot 1. The compaction installs a freshly CLONED -/// keys array (`object/delete_rest.rs` — `js_array_alloc` + -/// `set_object_keys_array`, cloned precisely because the old array is shared -/// between every instance of the shape), so a pointer compare against the -/// class's `@perry_class_keys_*` global catches it. The check is deliberately +/// moves `c` from slot 2 to slot 1. The compaction publishes a semantic +/// successor descriptor, so a ShapeId compare against the class's +/// `@perry_class_shape_id_*` global catches it. The check is deliberately /// DYNAMIC: the `delete` shape barrier that stands the analysis down is /// module-scoped while receivers alias across modules (#7143), so no static /// proof is available at this site. @@ -369,12 +355,12 @@ pub(crate) fn emit_class_field_loop_preheader_check( /// where the spec requires a strict-mode `TypeError`. The clone's own /// admission rules this out only through a MODULE-scoped freeze-barrier kill, /// so the receiver is vetted here as well. -/// * **Not-forwarded** and **`object_type == OBJECT_TYPE_REGULAR`** — the two +/// * **Not-forwarded**, **`GC_TYPE_OBJECT`**, and **not a class object** — the /// header predicates `js_object_get_class_id` does not itself check. /// /// Cost: one volatile `i8` load of the latch, three loads off the receiver (two /// of them from the `GcHeader` word the tower's class-id read already pulled -/// in), nine ALU ops and one conditional branch. `expected_keys` is expected to +/// in), nine ALU ops and one conditional branch. `expected_shape_id` is expected to /// come from an entry-hoisted slot (`LlFunction::entry_init_load_global`), so /// the global itself is read once per function, not per call. /// @@ -383,7 +369,7 @@ pub(crate) fn emit_class_field_loop_preheader_check( pub(crate) fn emit_proven_shape_recheck( ctx: &mut FnCtx, obj_handle: &str, - expected_keys: &str, + expected_shape_id: &str, proven_label: &str, generic_label: &str, ) { @@ -409,19 +395,15 @@ pub(crate) fn emit_proven_shape_recheck( let latched = blk.and(I16, &reserved, OBJ_FLAG_FROZEN_OR_DESCRIPTORS); let unlatched = blk.icmp_eq(I16, &latched, "0"); - // ObjectHeader: object_type @0 (i32), keys_array @16 (i64). `class_id` @4 - // was already matched by the tower's `js_object_get_class_id` compare. - let object_type = blk.load(I32, &obj_ptr); - let ot_ok = blk.icmp_eq(I32, &object_type, OBJECT_TYPE_REGULAR); - - let ka_ptr = blk.gep(I8, &obj_ptr, &[(I64, "16")]); - let keys_array = blk.load(I64, &ka_ptr); - let ka_ok = blk.icmp_eq(I64, &keys_array, expected_keys); + // `class_id` @4 was already matched by the tower. ShapeId @8 proves the + // exact immutable layout and receiver-kind descriptor. + let sid_ptr = blk.gep(I8, &obj_ptr, &[(I64, "8")]); + let shape_id = blk.load(I32, &sid_ptr); + let shape_ok = blk.icmp_eq(I32, &shape_id, expected_shape_id); let mut acc = blk.and(I1, &flag_ok, ¬_fwd); acc = blk.and(I1, &acc, &unlatched); - acc = blk.and(I1, &acc, &ot_ok); - acc = blk.and(I1, &acc, &ka_ok); + acc = blk.and(I1, &acc, &shape_ok); blk.cond_br(&acc, proven_label, generic_label); } @@ -454,8 +436,7 @@ pub(crate) fn emit_class_field_inline_precheck( obj_bits: &str, obj_handle: &str, expected_class_id: &str, - expected_keys: &str, - field_index: u32, + expected_shape_id: &str, require_raw_f64: bool, set_value_bits: Option<&str>, fast_label: &str, @@ -465,7 +446,6 @@ pub(crate) fn emit_class_field_inline_precheck( let guardcall_idx = ctx.new_block("class_field_inline.guardcall"); let deref_label = ctx.block_label(deref_idx); let guardcall_label = ctx.block_label(guardcall_idx); - let field_index_str = field_index.to_string(); // Gate the dereference: a basic block has no short-circuit, so the field // loads below must only run once we know (a) the inline path is enabled and @@ -511,27 +491,18 @@ pub(crate) fn emit_class_field_inline_precheck( let res_ptr = blk.gep(I8, &obj_ptr, &[(I64, "-6")]); let reserved = blk.load(I16, &res_ptr); - // ObjectHeader: object_type @0 (i32)==REGULAR, class_id @4 (i32), - // field_count @12 (i32), keys_array @16 (i64). - let object_type = blk.load(I32, &obj_ptr); - let ot_ok = blk.icmp_eq(I32, &object_type, OBJECT_TYPE_REGULAR); - + // ObjectHeader: class_id @4, authoritative ShapeId @8. let cid_ptr = blk.gep(I8, &obj_ptr, &[(I64, "4")]); let class_id = blk.load(I32, &cid_ptr); let cid_ok = blk.icmp_eq(I32, &class_id, expected_class_id); - let fc_ptr = blk.gep(I8, &obj_ptr, &[(I64, "12")]); - let field_count = blk.load(I32, &fc_ptr); - let fc_ok = blk.icmp_ugt(I32, &field_count, &field_index_str); - - let ka_ptr = blk.gep(I8, &obj_ptr, &[(I64, "16")]); - let keys_array = blk.load(I64, &ka_ptr); - let ka_ok = blk.icmp_eq(I64, &keys_array, expected_keys); + let sid_ptr = blk.gep(I8, &obj_ptr, &[(I64, "8")]); + let shape_id = blk.load(I32, &sid_ptr); + let sid_ok = blk.icmp_eq(I32, &shape_id, expected_shape_id); // (The process-global enable flag was already checked at the gate above, // before this dereference.) let mut acc = blk.and(I1, >ype_ok, ¬_fwd); - acc = blk.and(I1, &acc, &ot_ok); if subclass_arms.is_empty() { // Byte-for-byte the pre-widening and-chain. A class with no // eligible subclass must emit IDENTICAL IR, so the corpus-wide @@ -539,24 +510,20 @@ pub(crate) fn emit_class_field_inline_precheck( // and-chain alone made 17 of 19 corpus binaries differ for no // behavioural reason). acc = blk.and(I1, &acc, &cid_ok); - acc = blk.and(I1, &acc, &fc_ok); - acc = blk.and(I1, &acc, &ka_ok); + acc = blk.and(I1, &acc, &sid_ok); } else { - // The declared class's own (class id, keys) pair, OR any subclass + // The declared class's own (class id, ShapeId) pair, OR any subclass // arm's. Each arm is a full pair — matching a class id without its - // canonical keys array would accept an instance that has since - // grown a property and no longer has the packed layout this slot - // index describes. - let mut shape_ok = blk.and(I1, &cid_ok, &ka_ok); + // canonical descriptor would accept a diverged layout. + let mut shape_ok = blk.and(I1, &cid_ok, &sid_ok); for arm in subclass_arms { let arm_cid_ok = blk.icmp_eq(I32, &class_id, &arm.class_id.to_string()); - let arm_keys = blk.load(I64, &format!("@{}", arm.keys_global)); - let arm_ka_ok = blk.icmp_eq(I64, &keys_array, &arm_keys); - let arm_ok = blk.and(I1, &arm_cid_ok, &arm_ka_ok); + let arm_shape = blk.load(I32, &format!("@{}", arm.shape_id_global)); + let arm_shape_ok = blk.icmp_eq(I32, &shape_id, &arm_shape); + let arm_ok = blk.and(I1, &arm_cid_ok, &arm_shape_ok); shape_ok = blk.or(I1, &shape_ok, &arm_ok); } acc = blk.and(I1, &acc, &shape_ok); - acc = blk.and(I1, &acc, &fc_ok); } // #5654: a receiver that has ever had a property / accessor descriptor diff --git a/crates/perry-codegen/src/expr/element_shape_guard.rs b/crates/perry-codegen/src/expr/element_shape_guard.rs index cf6af16885..79805bf8ba 100644 --- a/crates/perry-codegen/src/expr/element_shape_guard.rs +++ b/crates/perry-codegen/src/expr/element_shape_guard.rs @@ -20,14 +20,15 @@ //! `js_array_ensure_element_shape` returning class id `C` means every element //! in `[0, verified_len)` passed `element_class_of_bits`: `POINTER_TAG`, a //! readable `GcHeader` (which rejects the handle bands and implausible -//! magnitudes), `obj_type == GC_TYPE_OBJECT`, `object_type == -//! OBJECT_TYPE_REGULAR`, and `class_id == C`. That is exactly the set of +//! magnitudes), `obj_type == GC_TYPE_OBJECT`, an exact ordinary-instance +//! ShapeId, +//! `class_id == C`. That is exactly the set of //! predicates the element-read tier and the *front half* of the class-field //! precheck spend per iteration, so the clone drops them. //! //! It proves nothing about the per-OBJECT facts a raw-f64 slot load needs — -//! `keys_array` identity (a `delete elem.f` compacts the packed slots while -//! preserving `class_id`), `field_count`, the per-object descriptor flag, or +//! exact ShapeId (a `delete elem.f` compacts the packed slots while preserving +//! `class_id`), the per-object descriptor flag, or //! the typed-layout intact bit. Those stay per element, but collapse to ONE //! 4-byte header load + two more loads and a single branch, because the three //! header bytes the check needs are contiguous. @@ -127,7 +128,7 @@ pub(crate) enum ElementShapeLoopTripCount<'a> { /// an allocation can move the array, so a base derived before it could be a /// from-space address. /// -/// Returns `(elements_base, expected_keys, shape_ok, bound_i32)`. +/// Returns `(elements_base, expected_shape_id, shape_ok, bound_i32)`. pub(crate) fn emit_element_shape_loop_preheader_check( ctx: &mut FnCtx, array_local_id: u32, @@ -255,7 +256,6 @@ pub(crate) fn emit_element_shape_loop_preheader_check( // clone is call-free, so THIS pointer is the pointer the clone uses. ctx.current_block = deref_idx; let arr1 = super::lower_expr(ctx, &perry_hir::Expr::LocalGet(array_local_id))?; - let keys_load = format!("@{keys_global_name}"); let blk = ctx.block(); let bits1 = blk.bitcast_double_to_i64(&arr1); let tag1 = blk.lshr(I64, &bits1, "48"); @@ -305,7 +305,8 @@ pub(crate) fn emit_element_shape_loop_preheader_check( // load is hoistable here for the same reason the class-field preheader // check hoists it: flipping it requires a runtime call, and the fast clone // makes none. - let expected_keys = blk.load(I64, &keys_load); + let shape_global = crate::typed_shape::shape_id_global_name_from_keys_global(keys_global_name); + let expected_shape_id = blk.load(I32, &format!("@{shape_global}")); let gate = blk.load_volatile(I8, "@PERRY_CLASS_FIELD_INLINE_GUARD_DISABLED"); let gate_ok = blk.icmp_eq(I8, &gate, "0"); @@ -315,7 +316,7 @@ pub(crate) fn emit_element_shape_loop_preheader_check( acc = blk.and(I1, &acc, &gate_ok); // No terminator: the caller branches after proving the clone call-free. - Ok((elements_base, expected_keys, acc, bound_i32)) + Ok((elements_base, expected_shape_id, acc, bound_i32)) } /// Emit one `arr[i].field` read inside the fast clone: bare element load, @@ -331,7 +332,6 @@ pub(crate) fn emit_element_shape_field_load( field_index: u32, ) -> String { let field_index_str = field_index.to_string(); - let max_field_index_str = fact.max_field_index.to_string(); let header_skip = crate::target_layout::object_header_size_bytes(ctx.target_triple).to_string(); let load_idx = ctx.new_block("element_shape.load"); @@ -356,16 +356,11 @@ pub(crate) fn emit_element_shape_field_load( let hdr_masked = blk.and(I32, &hdr, ELEM_HEADER_MASK); let hdr_ok = blk.icmp_eq(I32, &hdr_masked, ELEM_HEADER_EXPECT); - let fc_ptr = blk.gep(I8, &elem_ptr, &[(I64, "12")]); - let field_count = blk.load(I32, &fc_ptr); - let fc_ok = blk.icmp_ugt(I32, &field_count, &max_field_index_str); + let sid_ptr = blk.gep(I8, &elem_ptr, &[(I64, "8")]); + let shape_id = blk.load(I32, &sid_ptr); + let shape_ok = blk.icmp_eq(I32, &shape_id, &fact.expected_shape_id); - let ka_ptr = blk.gep(I8, &elem_ptr, &[(I64, "16")]); - let keys_array = blk.load(I64, &ka_ptr); - let ka_ok = blk.icmp_eq(I64, &keys_array, &fact.expected_keys); - - let mut ok = blk.and(I1, &hdr_ok, &fc_ok); - ok = blk.and(I1, &ok, &ka_ok); + let ok = blk.and(I1, &hdr_ok, &shape_ok); // One branch per access. The side exit resumes the CURRENT iteration // in the slow clone; the matcher guarantees no effect of this // iteration has committed yet, so re-executing cannot double-apply. diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index 2411599a85..9645beefbe 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -1648,8 +1648,8 @@ pub(crate) struct ElementShapeLoopFact { /// SSA name of the elements base pointer (`arr_handle + 8`), derived in /// the preheader AFTER the guard call, so it cannot be a pre-move address. pub elements_base: String, - /// SSA name of the hoisted `@perry_class_keys_` load. - pub expected_keys: String, + /// SSA name of the hoisted canonical ShapeId load. + pub expected_shape_id: String, /// Slow clone's preheader label. The per-element residual check (see /// `expr::element_shape_guard`) branches here on a miss; the slow clone /// re-executes the current iteration, which is safe because the matcher @@ -1658,9 +1658,6 @@ pub(crate) struct ElementShapeLoopFact { /// property name -> packed slot index, every entry a declared raw-f64 /// candidate validated by the matcher. pub fields: std::collections::BTreeMap, - /// Largest packed slot index the loop touches — the per-element - /// `field_count` check covers every tracked access with one compare. - pub max_field_index: u32, /// #7771: the body's `const r = arr[counter]` binding, when the matcher /// admitted the element-binding form. Inside the fast clone the `Let` /// itself emits nothing (`stmt/let_stmt.rs`) and every `r.field` read diff --git a/crates/perry-codegen/src/expr/property_get.rs b/crates/perry-codegen/src/expr/property_get.rs index cb892b4347..e5c6d3bd27 100644 --- a/crates/perry-codegen/src/expr/property_get.rs +++ b/crates/perry-codegen/src/expr/property_get.rs @@ -1625,6 +1625,11 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { .as_ref() .is_some_and(crate::typed_shape::type_is_raw_f64_candidate); let requires_raw_f64_str = if requires_raw_f64 { "1" } else { "0" }; + let expected_shape_id = crate::typed_shape::load_class_shape_id( + ctx, + &class_name, + &keys_global_name, + ); // #5391 path 2: oversized modules full-outline the entire // class-field-GET diamond (guard + fast load + fallback + // phi) to a single `js_class_field_get_ic(...)` call that @@ -1633,14 +1638,11 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // (the per-function compile time is superlinear in size). // Mirrors the field-SET full-outline (#5334 lever B). if crate::codegen::full_outline_ic_enabled() { - let (key_raw, expected_keys) = { + let key_raw = { let blk = ctx.block(); let key_box = blk.load(DOUBLE, &key_handle_global); let key_bits = blk.bitcast_double_to_i64(&key_box); - let key_raw = blk.and(I64, &key_bits, POINTER_MASK_I64); - let expected_keys = - blk.load(I64, &format!("@{}", keys_global_name)); - (key_raw, expected_keys) + blk.and(I64, &key_bits, POINTER_MASK_I64) }; let val = ctx.block().call( DOUBLE, @@ -1649,7 +1651,7 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { (I64, &site_id), (DOUBLE, &recv_box), (I32, &expected_class_id_str), - (I64, &expected_keys), + (I32, &expected_shape_id), (I64, &key_raw), (I32, &field_idx_str), (I32, requires_raw_f64_str), @@ -1660,15 +1662,14 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // #5093: build the guard operands once, up front, so both // the inline shape pre-check and the guard-call fallback // can reference them. - let (obj_bits, obj_handle, key_raw, expected_keys) = { + let (obj_bits, obj_handle, key_raw) = { let blk = ctx.block(); let obj_bits = blk.bitcast_double_to_i64(&recv_box); let obj_handle = blk.and(I64, &obj_bits, POINTER_MASK_I64); let key_box = blk.load(DOUBLE, &key_handle_global); let key_bits = blk.bitcast_double_to_i64(&key_box); let key_raw = blk.and(I64, &key_bits, POINTER_MASK_I64); - let expected_keys = blk.load(I64, &format!("@{}", keys_global_name)); - (obj_bits, obj_handle, key_raw, expected_keys) + (obj_bits, obj_handle, key_raw) }; let fast_idx = ctx.new_block("class_field_get.fast"); let fallback_idx = ctx.new_block("class_field_get.fallback"); @@ -1695,8 +1696,7 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { &obj_bits, &obj_handle, &expected_class_id_str, - &expected_keys, - field_index, + &expected_shape_id, requires_raw_f64, None, &fast_label, @@ -1709,7 +1709,7 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { (I64, &site_id), (DOUBLE, &recv_box), (I32, &expected_class_id_str), - (I64, &expected_keys), + (I32, &expected_shape_id), (I64, &key_raw), (I32, &field_idx_str), (I32, requires_raw_f64_str), diff --git a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs index 0c5c390efd..613081fc7a 100644 --- a/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs +++ b/crates/perry-codegen/src/expr/property_get/generic_dispatch.rs @@ -30,30 +30,10 @@ pub(crate) const PIC_WAY_BASE: usize = 4; /// shapes inline. Mirrors the runtime's `PIC_WAYS`. pub(crate) const PIC_WAYS: usize = 4; /// Way-state word: `> 0` means at least one way is populated and the compares -/// are worth running; `0` (fresh / epoch-wiped) and `-1` (sticky megamorphic) +/// are worth running; `0` (fresh) and a negative megamorphic countdown /// both skip them. Mirrors the runtime's `PIC_WAY_STATE`. pub(crate) const PIC_WAY_STATE: usize = 3; -/// `slot < max(field_count, INLINE_SLOT_FLOOR)` — the per-receiver -/// inline-capacity bound both the MRU hit path and the polymorphic ways apply -/// to a cached slot (#6804). -/// -/// Spelled as the equivalent disjunction `slot < FLOOR || slot < field_count` -/// rather than as a `max` followed by one compare. The predicate is identical -/// for every input (`x < max(a, b)` ⟺ `x < a ∨ x < b`), but the `max` had to be -/// materialised — `mov w, #FLOOR` / `cmp` / `csel` — and that `csel` was the -/// single hottest instruction in `interp.ts` (4.65% of `evalNode`, #7907), -/// because it sits on the dependency chain out of the `field_count` load. The -/// disjunction has no such node: LLVM folds the pair into `cmp` + `ccmp`, and -/// the `slot < FLOOR` half does not depend on the load at all. -fn emit_slot_in_bounds(ctx: &mut FnCtx<'_>, slot: &str, field_count: &str) -> String { - let below_floor = ctx - .block() - .icmp_ult(I64, slot, crate::target_layout::INLINE_SLOT_FLOOR_LIT); - let below_count = ctx.block().icmp_ult(I64, slot, field_count); - ctx.block().or(I1, &below_floor, &below_count) -} - /// The generic per-site monomorphic inline-cache dispatch for `obj.property`. /// This is the fall-through tail of the general catch-all arm: all earlier /// specializations have been ruled out. @@ -273,11 +253,10 @@ pub(crate) fn lower_generic_property_get( ctx.current_block = not_string_idx; } - // Issue #51: monomorphic inline cache. Per-site `[8 x i64]` global - // holds [shape_token, cached_slot_index, primed_epoch, ...unused]. - // The fast path compares the receiver's discriminated shape token - // (#6804: ShapeId stamp or raw keys_array pointer) to cache[0]; on - // match — pointer tokens additionally epoch-gated, #6080a — loads + // Monomorphic inline cache. The per-site global holds an authoritative + // ShapeId token and its cached slot; word 2 is non-identity scratch. + // The fast path compares the receiver's discriminated ShapeId token to + // cache[0] and, on match, loads // the field directly at obj+24+slot*8: no function call, no hash, // no linear scan. On miss, calls the slow helper which does the // full lookup and primes the cache for next time. @@ -289,18 +268,14 @@ pub(crate) fn lower_generic_property_get( ctx.ic_globals.push(cache_name.clone()); // Issue #72: validate the receiver is actually a GC_TYPE_OBJECT - // before treating offset 16 as `keys_array`. The v0.5.78 receiver + // before reading its ShapeId. The receiver // guard (`obj_handle > 0x100000`) keeps non-pointer NaN-boxes out, // but real heap pointers to Arrays/Strings/Buffers all clear that // threshold. A chained `obj.rowsRaw.length` (whose static type // analysis can't prove `obj.rowsRaw` is an Array — the outer // PropertyGet falls into this generic dispatch) hands the array's - // pointer to this PIC. For an Array, offset 16 is element[1]; on - // a freshly-allocated array element[1] is zero, the per-site - // cache global is zero-initialized, so the keys_val comparison - // falsely "hits" and the hit-path loads (obj+24+slot*8) — i.e. - // element[2] — as the field value, returning 0 instead of - // dispatching `.length`. The slow `js_object_get_field_by_name` + // pointer to this PIC. Reading an ObjectHeader ShapeId from that payload + // would be invalid. The slow `js_object_get_field_by_name` // already routes by `gc_type` (handles Array.length, String.length, // Set.size, Buffer.length, Error.message, etc.), so funneling // non-OBJECT receivers through the miss handler fixes correctness @@ -311,7 +286,7 @@ pub(crate) fn lower_generic_property_get( // ...) are NaN-boxed POINTER values whose lower-48 is a // small registry id (1, 2, 3, ...). The PIC fast path // below deref's `obj_handle - 8` for the GcHeader byte - // and `obj_handle + 16` for the keys_array slot — both + // and `obj_handle + 8` for the ShapeId slot — both // SIGSEGV when `obj_handle` is a small int. Funnel // small-handle receivers through the slow path so they // reach the runtime's `HANDLE_PROPERTY_DISPATCH` table @@ -319,26 +294,6 @@ pub(crate) fn lower_generic_property_get( // `req.params`, etc.). // // Threshold matches `js_native_call_method`'s small-handle - // detection (raw_ptr < 0x100000) and `js_object_get_field_by_name`'s - // post-#340 fix that calls HANDLE_PROPERTY_DISPATCH for - // these receivers. - // Issue #340/#341: small-handle guard. Receivers from - // native modules (axios, fastify, ioredis, better-sqlite3, - // ...) are NaN-boxed POINTER values whose lower-48 is a - // small registry id (1, 2, 3, ...). The PIC fast path - // below deref's `obj_handle - 8` for the GcHeader byte - // and `obj_handle + 16` for the keys_array slot — both - // SIGSEGV when `obj_handle` is a small int. Use a select - // to swap in a known-safe address (the per-site cache - // global itself) for the load, then AND `is_real_ptr` - // into the hit predicate so handle receivers cleanly - // miss to the slow path. The slow path - // (`js_object_get_field_ic_miss` → - // `js_object_get_field_by_name`) routes handles to - // `HANDLE_PROPERTY_DISPATCH` (axios `r.status` / `r.data`, - // fastify `req.query`, etc.). - // - // Threshold matches `js_native_call_method`'s small-handle // detection (raw_ptr < 0x100000). let cache_ref = format!("@{}", cache_name); let is_real_ptr = ctx.block().icmp_ugt(I64, &obj_handle, "1048575"); // 0x100000 @@ -346,8 +301,8 @@ pub(crate) fn lower_generic_property_get( // #7883: the hit/miss/merge blocks are minted here so the guard chain // below can BRANCH OUT to the miss on the first failing predicate // instead of AND-ing eight of them into one flat `hit`. LLVM if-converts - // a flat predicate, so every receiver paid every load and every compare — - // including the two epoch loads — even after the very first one had + // a flat predicate, so every receiver paid every load and every compare + // even after the very first one had // already decided the answer. Each group now ends in its own `cond_br`; // the miss block reconstructs what the polymorphic-way compares need // through phis (`false`/`0` on the early-exit edges, which is exactly @@ -373,9 +328,7 @@ pub(crate) fn lower_generic_property_get( // dereferenced. Pre-#7883 they were kept out of the loads by selecting a // sentinel address and AND-ing `is_real_ptr` into `hit`; the branch does // the same job without putting a `select` (and the sentinel's address - // materialisation) in front of every real object read. The miss path - // still substitutes the sentinel, because the way compares below load - // `field_count` unconditionally. + // materialisation) in front of every real object read. // A small-handle receiver can never resolve a way (`way_hit` requires a // real object), so it leaves for `pic.miss.cold` and never enters the // block the ways live in. @@ -392,42 +345,9 @@ pub(crate) fn lower_generic_property_get( let gc_type = ctx.block().load(I8, &gc_type_ptr); let is_object = ctx.block().icmp_eq(I8, &gc_type, "2"); - // Issue #618: closures share GC_TYPE_OBJECT but their offset+16 - // is a capture slot, not `keys_array`. The PIC's keys_val == - // cached_keys check would spuriously hit (per-site cache global - // is zero-initialized; capture[0] of a 0-capture wrapper is also - // often zero) and the hit path would load garbage from the - // capture region. Detect CLOSURE_MAGIC at +12 and force the - // PIC to miss for closures so the read routes through - // `js_object_get_field_ic_miss` → `js_object_get_field_by_name`, - // which dispatches closure dynamic-prop reads via the - // `CLOSURE_DYNAMIC_PROPS` side-table. - let magic_addr = ctx.block().add(I64, &obj_handle, "12"); - let magic_ptr = ctx.block().inttoptr(I64, &magic_addr); - let magic_val = ctx.block().load(I32, &magic_ptr); - // CLOSURE_MAGIC = 0x434C4F53 (4 bytes "CLOS" little-endian). - let is_closure = ctx.block().icmp_eq(I32, &magic_val, "1129268819"); - let not_closure = ctx.block().xor(I1, &is_closure, "true"); - let is_object = ctx.block().and(I1, &is_object, ¬_closure); - - // Issue #637: RegExpHeader / PromiseHeader / MapHeader / SetHeader - // / TypedArrayHeader / ... all share GC_TYPE_OBJECT but have - // different layouts than ObjectHeader. The first u32 of an - // ObjectHeader is `object_type = OBJECT_TYPE_REGULAR (=1)`; - // for these other headers the first 4 bytes are part of a - // pointer or method table, almost never 1. Without this check, - // a PIC site that learned a real ObjectHeader's [keys_array, - // slot] cache could spuriously hit on a regex/promise/etc. - // whose offset-16 happens to match (e.g. both null flags_ptr - // and uninitialized cache[0] are 0), and the hit path would - // load garbage from offset 24 of the non-Object header. - // Specific repro: `function f(): any { ... return new - // RegExp(...) } const r = f(); r.source` — fast path returns - // garbage f64 instead of routing through `js_regexp_get_source`. - let object_type_ptr = ctx.block().inttoptr(I64, &obj_handle); - let object_type = ctx.block().load(I32, &object_type_ptr); - let object_type_ok = ctx.block().icmp_eq(I32, &object_type, "1"); - let is_object = ctx.block().and(I1, &is_object, &object_type_ok); + // Closures and RegExp values have distinct GC kinds. Every + // `GC_TYPE_OBJECT` payload is therefore an ObjectHeader and its ShapeId is + // the remaining exact layout discriminator. // #6080: a receiver that has ever had a property/accessor descriptor // installed (`Object.defineProperty`) needs descriptor-aware dispatch — @@ -448,12 +368,12 @@ pub(crate) fn lower_generic_property_get( let no_desc = ctx.block().icmp_eq(crate::types::I16, &has_desc, "0"); let is_object = ctx.block().and(I1, &is_object, &no_desc); - // #7883: first exit. The four header predicates above are kept as one - // flat `and` on purpose — they are four loads from the SAME two cache - // lines and LLVM fuses their compares into a `ccmp` chain, which is + // #7883: first exit. The header predicates above are kept as one flat + // `and` on purpose — they are loads from the same cache line and LLVM + // fuses their compares into a `ccmp` chain, which is // cheaper than four branches. What was NOT worth folding is everything - // below: the keys load, the token select and the two epoch loads all - // hang off the same predicate, so a non-object receiver used to execute + // below: the ShapeId load and token select hang off the same predicate, + // so a non-object receiver used to execute // them before the flat `hit` could reject it. // // #7907: the false edge goes to `pic.miss.cold`, not `pic.miss` — a @@ -464,22 +384,8 @@ pub(crate) fn lower_generic_property_get( ctx.block().cond_br(&is_object, &tok_label, &cold_label); ctx.current_block = tok_idx; - // Load obj->keys_array at offset 16 of ObjectHeader. - let keys_addr = ctx.block().add(I64, &obj_handle, "16"); - let keys_ptr_p = ctx.block().inttoptr(I64, &keys_addr); - let keys_val = ctx.block().load(I64, &keys_ptr_p); - - // #6804: the receiver's shape TOKEN. A plain object stamped with a - // runtime ShapeId (`parent_class_id` ∈ [0x8000_0000, 0xC000_0000) — - // see `shapes::SHAPE_ID_BASE/END`; a real parent class id can never - // fall in that range) compares by id: stable across keys grow-reallocs - // and GC moves, and immune to address recycling (ids are never - // reused). Everything else (class instances, unstamped receivers) - // keeps the keys-pointer compare. Id tokens are lifted above the - // 48-bit pointer space (bit 62, `shapes::PIC_ID_TOKEN_BIT`) so the - // two token kinds can never collide numerically — one compare, no - // discriminant word. `parent_class_id` is a u32 at offset 8 on every - // target (the four leading u32s precede the pointer fields). + // The receiver token is derived solely from its authoritative ShapeId. + // Invalid/unstamped payloads produce zero and miss closed. let pcid_addr = ctx.block().add(I64, &obj_handle, "8"); let pcid_ptr = ctx.block().inttoptr(I64, &pcid_addr); let pcid = ctx.block().load(I32, &pcid_ptr); @@ -490,7 +396,7 @@ pub(crate) fn lower_generic_property_get( let pcid64 = ctx.block().zext(I32, &pcid, I64); // PIC_ID_TOKEN_BIT = 1 << 62. let id_token = ctx.block().or(I64, &pcid64, "4611686018427387904"); - let token = ctx.block().select(I1, &is_stamp, I64, &id_token, &keys_val); + let token = ctx.block().select(I1, &is_stamp, I64, &id_token, "0"); // Load the cached token from the per-site global. let cache_keys_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "0")]); @@ -510,52 +416,15 @@ pub(crate) fn lower_generic_property_get( let token_nonnull = ctx.block().icmp_ne(I64, &token, "0"); let hit = ctx.block().and(I1, &token_eq, &token_nonnull); - // #6080a: pointer tokens are only trustworthy within the GC epoch they - // were primed in. The `@perry_ic_N` global is invisible to every GC - // scanner, so after a collection frees or evacuates a shape-shared keys - // array, its recycled address can be adopted by a different-shape keys - // array — `token_eq` then falsely matches and the hit path loads the - // wrong slot, silently. `js_object_get_field_ic_miss` snapshots - // `PERRY_IC_EPOCH` into `cache[2]` at prime time and every completed - // collection bumps the global, so requiring `cache[2] == PERRY_IC_EPOCH` - // forces the first read after any collection back through the miss - // handler (which re-primes against live arrays). Shape-ID tokens - // (`is_stamp`, #6804) bypass the check — ids are never reused, so they - // cannot alias across collections. Cost on the hot stamped path: two - // loads + icmp + or, folded into the existing `hit` cond_br. - let cache_epoch_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "2")]); - let cache_epoch = ctx.block().load(I64, &cache_epoch_ptr); - let live_epoch = ctx.block().load(I64, "@PERRY_IC_EPOCH"); - let epoch_eq = ctx.block().icmp_eq(I64, &cache_epoch, &live_epoch); - let epoch_ok = ctx.block().or(I1, &is_stamp, &epoch_eq); - let hit = ctx.block().and(I1, &hit, &epoch_ok); - ctx.block().cond_br(&hit, &hit_label, &miss_label); - // PIC hit: bounds-check the cached slot, then direct field load. + // `js_object_get_field_ic_miss` primes only slots below the descriptor's + // exact `live_inline_slot_count`. ShapeIds are never reused, so an exact + // token hit permanently proves that the cached slot remains live and + // makes the raw load below safe without a compatibility-header bound. ctx.current_block = hit_idx; let cache_slot_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "1")]); let slot = ctx.block().load(I64, &cache_slot_ptr); - // #6804: bound the cached slot by THIS receiver's inline capacity. - // Same-shape siblings can differ in physical allocation (an object - // built at a small alloc site adopts a shared keys array whose later - // slots live in its OVERFLOW map) — a slot primed from a - // larger-capacity sibling must not drive a raw load past this - // receiver's field region. `alloc_limit = max(field_count, - // INLINE_SLOT_FLOOR)` mirrors the miss handler's cacheability - // rule; an out-of-bounds slot falls to the miss path, which reads - // the overflow map correctly (and records the guard failure — - // `record_guard_pass` only fires after the bounds check passes). - let fc_addr = ctx.block().add(I64, &obj_handle, "12"); - let fc_ptr = ctx.block().inttoptr(I64, &fc_addr); - let fc = ctx.block().load(I32, &fc_ptr); - let fc64 = ctx.block().zext(I32, &fc, I64); - let slot_in_bounds = emit_slot_in_bounds(ctx, &slot, &fc64); - let bounds_hit = ctx.new_block("pic.hit.load"); - let bounds_hit_label = ctx.block_label(bounds_hit); - ctx.block() - .cond_br(&slot_in_bounds, &bounds_hit_label, &miss_label); - ctx.current_block = bounds_hit; crate::expr::emit_typed_feedback_record_call( ctx.block(), "js_typed_feedback_record_guard_pass", @@ -598,24 +467,20 @@ pub(crate) fn lower_generic_property_get( // // # Why this block is DOMINATED by `pic.token` (#7907) // - // Its only predecessors are `pic.token` (the MRU token did not match) and - // `pic.hit` (it matched but the cached slot is outside this receiver's - // inline capacity), and `pic.hit` is itself dominated by `pic.token`. So - // `token`, `token_nonnull` and `epoch_eq` — everything the way compares - // need — are already in scope here, and `is_object` is statically TRUE. + // Its only predecessor is `pic.token` after the MRU token did not match. + // The exact descriptor identity proves cached-slot bounds, so `token` and + // `token_nonnull` are everything the way compares need. // // #7883 could not rely on that: it routed the two receiver-validation // failures here as well, which left the values live on only some edges, so - // the block **re-derived them** — four header loads, the `keys_array` and - // `parent_class_id` loads, the token select, a second pair of epoch loads, - // and a `select` substituting a safe address for a small-handle receiver. + // the block **re-derived them** — header and identity loads, the token + // select, and a safe-address select for small-handle receivers. // That was correct, and it was justified as cold. It is not cold: on a site // whose receiver rotates over more shapes than the MRU entry holds — the // shape #7753's ways exist for — this block runs on nearly every read, so // the duplicate ladder sat on the hot path. Measured on `interp.ts`'s // `evalNode`, the single hottest instruction in the whole program was the - // `csel` materialising `max(field_count, INLINE_SLOT_FLOOR)` *inside this - // recomputation*. + // redundant receiver reconstruction inside this block. // // Sending the two validation failures to `pic.miss.cold` instead is what // establishes the dominance. Nothing about the predicate changed: a @@ -634,21 +499,9 @@ pub(crate) fn lower_generic_property_get( &[(I64, &feedback_site_id)], ); - // A way can hold EITHER token kind, so it carries the pointer-token - // guarantees: `cache[2] == @PERRY_IC_EPOCH` (`epoch_eq`, already computed - // for the MRU predicate) plus a non-zero receiver token so an empty way - // (0) can never match a keyless receiver whose `keys_array` is also 0 - // (#809's shape, applied to the ways). `pic_prime_get` wipes every way - // whenever it writes a new epoch into word 2, so one shared epoch word - // covers all of them: a readable way was necessarily primed in the epoch - // that word still holds. - // - // Restricting the ways to shape-ID tokens instead — which needs no epoch - // guard at all, ids being unreusable — looks safer and is useless: a plain - // object literal is built by a generated `__AnonShape_*` constructor and - // therefore has a real `class_id`, which primes the keys-POINTER token. An - // ID-only way set never fills for the discriminated-union programs this - // whole block exists to speed up; measured, it cost 6%. + // Every way contains a ShapeId token. A non-zero receiver token keeps an + // empty way from matching; no GC-epoch guard is necessary because ids are + // never reused and descriptor identity survives key relocation. // // The compares sit behind their own branch on `cache[PIC_WAY_STATE] > 0` // rather than being folded into one flat predicate, because a site whose @@ -656,7 +509,7 @@ pub(crate) fn lower_generic_property_get( // otherwise pay four dependent loads on every read: measured at **+37%** on // a 7-shape site, against a 2.5x speedup on a 5-shape one. `pic_prime_get` // latches that state to `-1` once a site proves itself megamorphic, and a - // fresh or epoch-wiped site reads `0`, so for both the branch is one load, + // fresh site reads `0`, so for both the branch is one load, // one compare, and a perfectly predicted fall-through to the call — which // is exactly the pre-#7753 code path. let state_ptr = ctx @@ -670,10 +523,10 @@ pub(crate) fn lower_generic_property_get( ctx.current_block = ways_idx; // `is_object` is not ANDed in any more: it is statically true on every edge - // that reaches here (#7907 — see the dominance note above). `epoch_eq` and - // `token_nonnull` are the values `pic.token` computed, from the same memory + // that reaches here (#7907 — see the dominance note above). + // `token_nonnull` is the value `pic.token` computed, from the same memory // with no intervening store, so the predicate is unchanged. - let mut way_hit = ctx.block().and(I1, &epoch_eq, &token_nonnull); + let mut way_hit = token_nonnull.clone(); // Reduced as a BALANCED TREE, not as a left fold. At most one way can hold // a given token (`pic_prime_get` evicts a duplicate before it writes one, // and a zero token is excluded by `token_nonnull`), so the association is @@ -718,22 +571,9 @@ pub(crate) fn lower_generic_property_get( .pop() .expect("PIC_WAYS is non-zero, so the reduction leaves exactly one lane"); way_hit = ctx.block().and(I1, &way_hit, &way_any); - // Same per-receiver inline-capacity bound the MRU hit path applies: a slot - // primed from a larger-capacity sibling of the same shape must not drive a - // raw load past this receiver's field region (#6804). - // - // The load is off `obj_handle` rather than the deleted small-handle - // sentinel, so it is the SAME address `pic.recv_hdr` already read for the - // closure-magic check and GVN folds the two together. - let way_fc_addr = ctx.block().add(I64, &obj_handle, "12"); - let way_fc_ptr = ctx.block().inttoptr(I64, &way_fc_addr); - let way_fc = ctx.block().load(I32, &way_fc_ptr); - let way_fc64 = ctx.block().zext(I32, &way_fc, I64); - let way_in_bounds = emit_slot_in_bounds(ctx, &way_slot, &way_fc64); - let way_ok = ctx.block().and(I1, &way_hit, &way_in_bounds); let way_load_idx = ctx.new_block("pic.way.load"); let way_load_label = ctx.block_label(way_load_idx); - ctx.block().cond_br(&way_ok, &way_load_label, &call_label); + ctx.block().cond_br(&way_hit, &way_load_label, &call_label); ctx.current_block = way_load_idx; let way_offset = ctx.block().shl(I64, &way_slot, "3"); diff --git a/crates/perry-codegen/src/expr/property_get/helpers.rs b/crates/perry-codegen/src/expr/property_get/helpers.rs index 79f8d28694..0528b4f27b 100644 --- a/crates/perry-codegen/src/expr/property_get/helpers.rs +++ b/crates/perry-codegen/src/expr/property_get/helpers.rs @@ -718,15 +718,16 @@ pub(crate) fn lower_raw_f64_class_field_get_for_number_context( ); let field_idx_str = field_index.to_string(); let expected_class_id_str = expected_class_id.to_string(); - let (obj_bits, obj_handle, key_raw, expected_keys) = { + let expected_shape_id = + crate::typed_shape::load_class_shape_id(ctx, &class_name, &keys_global_name); + let (obj_bits, obj_handle, key_raw) = { let blk = ctx.block(); let obj_bits = blk.bitcast_double_to_i64(&recv_box); let obj_handle = blk.and(I64, &obj_bits, POINTER_MASK_I64); let key_box = blk.load(DOUBLE, &key_handle_global); let key_bits = blk.bitcast_double_to_i64(&key_box); let key_raw = blk.and(I64, &key_bits, POINTER_MASK_I64); - let expected_keys = blk.load(I64, &format!("@{}", keys_global_name)); - (obj_bits, obj_handle, key_raw, expected_keys) + (obj_bits, obj_handle, key_raw) }; let fast_idx = ctx.new_block("class_field_get_number.fast"); @@ -748,8 +749,7 @@ pub(crate) fn lower_raw_f64_class_field_get_for_number_context( &obj_bits, &obj_handle, &expected_class_id_str, - &expected_keys, - field_index, + &expected_shape_id, true, None, &fast_label, @@ -762,7 +762,7 @@ pub(crate) fn lower_raw_f64_class_field_get_for_number_context( (I64, &site_id), (DOUBLE, &recv_box), (I32, &expected_class_id_str), - (I64, &expected_keys), + (I32, &expected_shape_id), (I64, &key_raw), (I32, &field_idx_str), (I32, "1"), diff --git a/crates/perry-codegen/src/expr/property_get/tests.rs b/crates/perry-codegen/src/expr/property_get/tests.rs index 34eda40c66..30a8572285 100644 --- a/crates/perry-codegen/src/expr/property_get/tests.rs +++ b/crates/perry-codegen/src/expr/property_get/tests.rs @@ -123,32 +123,27 @@ fn no_call_location_without_debug_symbols() { ); } -/// #6080a: the inline PIC hit predicate must gate raw keys-POINTER tokens on -/// the GC epoch — `cache[2] == @PERRY_IC_EPOCH` — because the `@perry_ic_N` -/// globals are invisible to every GC scanner, so a primed keys-array address -/// that GC frees/moves can be recycled under a different shape and falsely -/// pointer-match. This asserts the emitted IR still carries the guard: the -/// per-site epoch-slot load (gep index 2) and the live-epoch load from the -/// runtime-exported global. Deleting either from `lower_generic_property_get` -/// turns this red. +/// #8067: property-read PIC identity is the authoritative ShapeId only. The +/// former keys-pointer epoch word is reserved scratch and must not participate +/// in the emitted hit predicate. #[test] -fn generic_property_get_hit_path_is_epoch_gated() { +fn generic_property_get_hit_path_is_shape_id_only() { let ir = emit(false, None); assert!( ir.contains("@perry_ic_"), "test premise: the generic read reaches the inline monomorphic PIC:\n{ir}" ); assert!( - ir.contains("load i64, ptr @PERRY_IC_EPOCH"), - "hit path must load the live read-PIC epoch (@PERRY_IC_EPOCH):\n{ir}" + ir.contains("4611686018427387904"), + "hit path must form a discriminated ShapeId token:\n{ir}" ); - // The per-site primed-epoch slot: a gep to index 2 of some @perry_ic_N - // global (the site number depends on how many IC sites precede this one). assert!( - ir.lines().any(|l| { - l.contains("getelementptr i64, ptr @perry_ic_") && l.trim_end().ends_with(", i64 2") - }), - "hit path must load the per-site primed-epoch slot (cache[2]):\n{ir}" + !ir.contains("@PERRY_IC_EPOCH") + && !ir.lines().any(|line| { + line.contains("getelementptr i64, ptr @perry_ic_") + && line.trim_end().ends_with(", i64 2") + }), + "the removed pointer-token epoch must not appear in emitted IR:\n{ir}" ); } @@ -274,7 +269,7 @@ fn generic_property_get_tries_ways_before_calling_the_miss_handler() { /// /// #7883 routed all four failure edges — small-handle receiver, non-object /// receiver, MRU token mismatch, cached slot out of bounds — into one block, -/// which left `token` / `token_nonnull` / `epoch_eq` live on only some of them +/// which left `token` / `token_nonnull` / `shape_id_eq` live on only some of them /// and forced the block to reload the whole header ladder. That block is not /// cold: on a receiver rotation wider than the MRU entry it runs on nearly /// every read, so the duplicate ladder was hot code. The fix is purely @@ -283,8 +278,7 @@ fn generic_property_get_tries_ways_before_calling_the_miss_handler() { /// the dominance follows. /// /// Assert the *consequences*, not the block names alone: a re-derivation would -/// show up as a second `@PERRY_IC_EPOCH` load and as the small-handle sentinel -/// `select`, and both must be gone. +/// show up as duplicate header loads or the small-handle sentinel `select`. #[test] fn pic_miss_reuses_the_token_blocks_values_instead_of_re_deriving_them() { let ir = emit(false, None); @@ -305,11 +299,9 @@ fn pic_miss_reuses_the_token_blocks_values_instead_of_re_deriving_them() { "the two receiver-validation failures need their own landing block, \ otherwise pic.miss is not dominated by pic.token:\n{ir}" ); - let epoch_loads = main.matches("load i64, ptr @PERRY_IC_EPOCH").count(); - assert_eq!( - epoch_loads, 1, - "one generic read must load @PERRY_IC_EPOCH exactly once; a second \ - load means the way block re-derived the epoch predicate:\n{ir}" + assert!( + !main.contains("@PERRY_IC_EPOCH"), + "the removed keys-pointer epoch global must not appear:\n{ir}" ); assert!( !main.contains("ptrtoint ptr @perry_ic_"), @@ -319,7 +311,7 @@ fn pic_miss_reuses_the_token_blocks_values_instead_of_re_deriving_them() { // The header predicates: each load/compare pair must appear exactly once. for (needle, what) in [ ("icmp eq i8 ", "the GC_TYPE_OBJECT compare"), - ("icmp eq i32 %", "the closure-magic / object_type compares"), + ("icmp eq i32 %", "the ShapeId identity compare"), ] { let n = main.matches(needle).count(); assert!( @@ -330,28 +322,22 @@ fn pic_miss_reuses_the_token_blocks_values_instead_of_re_deriving_them() { } } -/// #7907: the cached-slot bound is `slot < FLOOR || slot < field_count`, not -/// `slot < max(field_count, FLOOR)`. -/// -/// Identical predicate; the point is that the `max` had to be materialised, and -/// the `csel` that did it sat on the dependency chain out of the `field_count` -/// load — the single hottest instruction in `interp.ts`'s `evalNode`. If -/// someone "simplifies" this back to a `max`, nothing else in the suite -/// notices. +/// #8067: an exact ShapeId match proves the cached slot's descriptor facts, so +/// the hit path must not reload the compatibility `field_count` mirror merely +/// to re-prove the slot bound. #[test] -fn cached_slot_bound_is_a_disjunction_not_a_materialised_max() { +fn cached_slot_bound_comes_from_the_shape_descriptor_match() { let floor = crate::target_layout::INLINE_SLOT_FLOOR_LIT; let ir = emit(false, None); assert!( - ir.lines() - .any(|l| l.contains("icmp ult i64 ") && l.ends_with(&format!(", {floor}"))), - "test premise: the emitted bound compares a slot against \ - INLINE_SLOT_FLOOR ({floor}):\n{ir}" + ir.contains("4611686018427387904") && ir.contains("@perry_ic_"), + "test premise: the emitted read uses a ShapeId PIC:\n{ir}" ); assert!( - !ir.contains(&format!(", i64 {floor}, i64 %")), - "a `select …, i64 {floor}, i64 %fc` is the materialised max this \ - deliberately does not emit:\n{ir}" + !ir.lines() + .any(|line| line.contains("icmp ult i64 ") && line.ends_with(&format!(", {floor}"))) + && !ir.contains(&format!(", i64 {floor}, i64 %")), + "the ShapeId hit path must not materialize a header slot bound:\n{ir}" ); } @@ -505,7 +491,7 @@ fn generic_property_get_slot_load_is_reached_only_through_every_guard() { // string-handle `ptrtoint` as the receiver-tag test before it was fixed). let func = ir .split("\ndefine ") - .find(|f| f.contains("pic.hit.load")) + .find(|f| f.contains("\npic.hit.") && f.contains("@perry_ic_")) .unwrap_or_else(|| panic!("no function contains a PIC hit load:\n{ir}")) .to_string(); @@ -531,9 +517,11 @@ fn generic_property_get_slot_load_is_reached_only_through_every_guard() { } let load_label = blocks .iter() - .find(|(l, _)| l.starts_with("pic.hit.load")) + .find(|(l, body)| { + l.starts_with("pic.hit") && body.iter().any(|line| line.contains("load double")) + }) .map(|(l, _)| l.clone()) - .unwrap_or_else(|| panic!("no `pic.hit.load` block:\n{func}")); + .unwrap_or_else(|| panic!("no `pic.hit*` block containing a slot load:\n{func}")); let mut defs: std::collections::HashMap = std::collections::HashMap::new(); for (_, body) in &blocks { @@ -598,8 +586,8 @@ fn generic_property_get_slot_load_is_reached_only_through_every_guard() { at = pred_label.clone(); } assert!( - conds.len() >= 5, - "expected at least five guard branches between the PIC entry and the \ + conds.len() >= 3, + "expected at least three guard branches between the PIC entry and the \ inline slot load, found {}: {conds:?}\n{func}", conds.len() ); @@ -636,9 +624,8 @@ fn generic_property_get_slot_load_is_reached_only_through_every_guard() { ("32765", "the POINTER/STRING receiver-tag test"), ("1048575", "the small-handle (native registry id) test"), ("icmp eq i8", "the GcHeader obj_type == GC_TYPE_OBJECT test"), - ("1129268819", "the CLOSURE_MAGIC test"), ("2048", "the OBJ_FLAG_HAS_DESCRIPTORS test"), - ("@PERRY_IC_EPOCH", "the read-PIC epoch gate"), + ("4611686018427387904", "the discriminated ShapeId token"), ("@perry_ic_", "the per-site cached shape-token compare"), ] { assert!( diff --git a/crates/perry-codegen/src/expr/property_set.rs b/crates/perry-codegen/src/expr/property_set.rs index 4b94159c5c..a8a651cca4 100644 --- a/crates/perry-codegen/src/expr/property_set.rs +++ b/crates/perry-codegen/src/expr/property_set.rs @@ -301,15 +301,16 @@ pub(crate) fn try_lower_sloppy_class_field_store( let key_handle_global = format!("@{}", ctx.strings.entry(key_idx).handle_global); let field_idx_str = field_index.to_string(); let expected_class_id_str = expected_class_id.to_string(); + let expected_shape_id = + crate::typed_shape::load_class_shape_id(ctx, &class_name, &keys_global_name); - let (obj_bits, obj_handle, key_box, val_bits, expected_keys) = { + let (obj_bits, obj_handle, key_box, val_bits) = { let blk = ctx.block(); let obj_bits = blk.bitcast_double_to_i64(&recv_box); let obj_handle = blk.and(I64, &obj_bits, POINTER_MASK_I64); let key_box = blk.load(DOUBLE, &key_handle_global); let val_bits = blk.bitcast_double_to_i64(&val_double); - let expected_keys = blk.load(I64, &format!("@{}", keys_global_name)); - (obj_bits, obj_handle, key_box, val_bits, expected_keys) + (obj_bits, obj_handle, key_box, val_bits) }; let fast_idx = ctx.new_block("class_field_sloppy_set.fast"); @@ -331,8 +332,7 @@ pub(crate) fn try_lower_sloppy_class_field_store( &obj_bits, &obj_handle, &expected_class_id_str, - &expected_keys, - field_index, + &expected_shape_id, true, Some(&val_bits), &fast_label, @@ -434,15 +434,16 @@ fn try_lower_sloppy_class_field_boxed_store( let key_handle_global = format!("@{}", ctx.strings.entry(key_idx).handle_global); let field_idx_str = field_index.to_string(); let expected_class_id_str = expected_class_id.to_string(); + let expected_shape_id = + crate::typed_shape::load_class_shape_id(ctx, class_name, keys_global_name); - let (obj_bits, obj_handle, key_box, val_bits, expected_keys) = { + let (obj_bits, obj_handle, key_box, val_bits) = { let blk = ctx.block(); let obj_bits = blk.bitcast_double_to_i64(&recv_box); let obj_handle = blk.and(I64, &obj_bits, POINTER_MASK_I64); let key_box = blk.load(DOUBLE, &key_handle_global); let val_bits = blk.bitcast_double_to_i64(&val_double); - let expected_keys = blk.load(I64, &format!("@{}", keys_global_name)); - (obj_bits, obj_handle, key_box, val_bits, expected_keys) + (obj_bits, obj_handle, key_box, val_bits) }; let fast_idx = ctx.new_block("class_field_sloppy_set.boxed_fast"); @@ -464,8 +465,7 @@ fn try_lower_sloppy_class_field_boxed_store( &obj_bits, &obj_handle, &expected_class_id_str, - &expected_keys, - field_index, + &expected_shape_id, false, Some(&val_bits), &fast_label, @@ -1287,16 +1287,18 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // reduction, so clang -O0 — which oversized modules are // forced to (#4880) — can actually compile the module. // Only the call's own operands are materialized (the key - // handle + expected-keys), not the inline-store scaffolding. + // handle + expected ShapeId), not the inline-store scaffolding. + let expected_shape_id = crate::typed_shape::load_class_shape_id( + ctx, + &class_name, + &keys_global_name, + ); if crate::codegen::full_outline_ic_enabled() { - let (key_raw, expected_keys) = { + let key_raw = { let blk = ctx.block(); let key_box = blk.load(DOUBLE, &key_handle_global); let key_bits = blk.bitcast_double_to_i64(&key_box); - let key_raw = blk.and(I64, &key_bits, POINTER_MASK_I64); - let expected_keys = - blk.load(I64, &format!("@{}", keys_global_name)); - (key_raw, expected_keys) + blk.and(I64, &key_bits, POINTER_MASK_I64) }; ctx.block().call_void( "js_class_field_set_ic", @@ -1304,7 +1306,7 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { (I64, &site_id), (DOUBLE, &recv_box), (I32, &expected_class_id_str), - (I64, &expected_keys), + (I32, &expected_shape_id), (I64, &key_raw), (I32, &field_idx_str), (DOUBLE, &val_double), @@ -1316,17 +1318,15 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // #5093: build the guard operands once, up front, so both // the inline shape pre-check and the guard-call fallback // can reference them. - let (obj_bits, obj_handle, key_raw, expected_keys, val_bits) = { + let (obj_bits, obj_handle, key_raw, val_bits) = { let blk = ctx.block(); let obj_bits = blk.bitcast_double_to_i64(&recv_box); let obj_handle = blk.and(I64, &obj_bits, POINTER_MASK_I64); let key_box = blk.load(DOUBLE, &key_handle_global); let key_bits = blk.bitcast_double_to_i64(&key_box); let key_raw = blk.and(I64, &key_bits, POINTER_MASK_I64); - let expected_keys = - blk.load(I64, &format!("@{}", keys_global_name)); let val_bits = blk.bitcast_double_to_i64(&val_double); - (obj_bits, obj_handle, key_raw, expected_keys, val_bits) + (obj_bits, obj_handle, key_raw, val_bits) }; let fast_idx = ctx.new_block("class_field_set.fast"); let fallback_idx = ctx.new_block("class_field_set.fallback"); @@ -1379,21 +1379,20 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // literal, so a candidate whose declared type disagrees // about the slot's representation is dropped. let subclass_arms = - crate::expr::class_field_inline_guard::class_field_subclass_arms( - ctx, - &class_name, - property, - field_index, - requires_raw_f64, - ); + crate::expr::class_field_inline_guard::class_field_subclass_arms( + ctx, + &class_name, + property, + field_index, + requires_raw_f64, + ); let _guardcall_label = crate::expr::class_field_inline_guard::emit_class_field_inline_precheck( ctx, &obj_bits, &obj_handle, &expected_class_id_str, - &expected_keys, - field_index, + &expected_shape_id, requires_raw_f64, Some(&val_bits), &fast_label, @@ -1406,7 +1405,7 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { (I64, &site_id), (DOUBLE, &recv_box), (I32, &expected_class_id_str), - (I64, &expected_keys), + (I32, &expected_shape_id), (I64, &key_raw), (I32, &field_idx_str), (DOUBLE, &val_double), diff --git a/crates/perry-codegen/src/expr/proxy_reflect.rs b/crates/perry-codegen/src/expr/proxy_reflect.rs index db80555f0c..84007ffc47 100644 --- a/crates/perry-codegen/src/expr/proxy_reflect.rs +++ b/crates/perry-codegen/src/expr/proxy_reflect.rs @@ -46,6 +46,10 @@ use super::{ unbox_str_handle, unbox_to_i64, FnCtx, }; +/// Runtime write-PIC flags that force the miss path. Class-vs-instance kind is +/// encoded by the authoritative ShapeId and therefore owns no header flag. +const WRITE_PIC_BLOCKING_FLAGS: u16 = 0x1907; + /// The NaN-boxed `undefined` literal, for an absent optional operand. fn undefined_literal() -> String { double_literal(f64::from_bits(crate::nanbox::TAG_UNDEFINED)) @@ -535,41 +539,31 @@ fn lower_put_value_static_write_ic( // Existing-own overwrite guards. Bit 12 is the per-object typed-layout // intact bit: the runtime miss downgrades it before priming this cache, so // same-shape siblings take one miss each before direct stores are allowed. - const BLOCKING_FLAGS: u16 = 0x1907; // frozen/sealed/noextend/TA-proto/descriptors/typed-intact let reserved_addr = ctx.block().sub(I64, &safe_target, "6"); let reserved_ptr = ctx.block().inttoptr(I64, &reserved_addr); let reserved = ctx.block().load(I16, &reserved_ptr); - let blocked = ctx.block().and(I16, &reserved, &BLOCKING_FLAGS.to_string()); + let blocked = ctx + .block() + .and(I16, &reserved, &WRITE_PIC_BLOCKING_FLAGS.to_string()); let flags_clear = ctx.block().icmp_eq(I16, &blocked, "0"); - let object_type_ptr = ctx.block().inttoptr(I64, &safe_target); - let object_type = ctx.block().load(I32, &object_type_ptr); - let regular = ctx.block().icmp_eq(I32, &object_type, "1"); let class_addr = ctx.block().add(I64, &safe_target, "4"); let class_ptr = ctx.block().inttoptr(I64, &class_addr); let class_id = ctx.block().load(I32, &class_ptr); let class_nonzero = ctx.block().icmp_ne(I32, &class_id, "0"); let not_native_module = ctx.block().icmp_ne(I32, &class_id, "-2"); - let keys_addr = ctx.block().add(I64, &safe_target, "16"); - let keys_ptr = ctx.block().inttoptr(I64, &keys_addr); - let keys = ctx.block().load(I64, &keys_ptr); - - // Mirror the read PIC's #6804 discriminated shape token. Plain objects - // carrying a never-reused runtime ShapeId compare by that stable id, - // lifted above the pointer range with bit 62. Class instances and - // unstamped receivers compare by their shared keys pointer. The runtime - // miss publishes the same token representation. - let parent_class_addr = ctx.block().add(I64, &safe_target, "8"); - let parent_class_ptr = ctx.block().inttoptr(I64, &parent_class_addr); - let parent_class_id = ctx.block().load(I32, &parent_class_ptr); - let shape_id_rel = ctx.block().add(I32, &parent_class_id, "-2147483648"); + // The write PIC uses the same single ShapeId token domain as the read PIC. + let shape_id_addr = ctx.block().add(I64, &safe_target, "8"); + let shape_id_ptr = ctx.block().inttoptr(I64, &shape_id_addr); + let raw_shape_id = ctx.block().load(I32, &shape_id_ptr); + let shape_id_rel = ctx.block().add(I32, &raw_shape_id, "-2147483648"); let has_shape_id = ctx.block().icmp_ult(I32, &shape_id_rel, "1073741824"); - let shape_id64 = ctx.block().zext(I32, &parent_class_id, I64); + let shape_id64 = ctx.block().zext(I32, &raw_shape_id, I64); let shape_id_token = ctx.block().or(I64, &shape_id64, "4611686018427387904"); let shape_token = ctx .block() - .select(I1, &has_shape_id, I64, &shape_id_token, &keys); + .select(I1, &has_shape_id, I64, &shape_id_token, "0"); let cached_token_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "0")]); let cached_token = ctx.block().load(I64, &cached_token_ptr); let token_match = ctx.block().icmp_eq(I64, &shape_token, &cached_token); @@ -577,27 +571,13 @@ fn lower_put_value_static_write_ic( let cached_slot_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "1")]); let slot = ctx.block().load(I64, &cached_slot_ptr); - let field_count_addr = ctx.block().add(I64, &safe_target, "12"); - let field_count_ptr = ctx.block().inttoptr(I64, &field_count_addr); - let field_count = ctx.block().load(I32, &field_count_ptr); - let field_count64 = ctx.block().zext(I32, &field_count, I64); - let below_floor = ctx - .block() - .icmp_ult(I64, &field_count64, INLINE_SLOT_FLOOR_LIT); - let inline_limit = - ctx.block() - .select(I1, &below_floor, I64, INLINE_SLOT_FLOOR_LIT, &field_count64); - let slot_in_bounds = ctx.block().icmp_ult(I64, &slot, &inline_limit); - let mut hit = ctx.block().and(I1, &heap_candidate, &gc_object); hit = ctx.block().and(I1, &hit, ¬_forwarded); hit = ctx.block().and(I1, &hit, &flags_clear); - hit = ctx.block().and(I1, &hit, ®ular); hit = ctx.block().and(I1, &hit, &class_nonzero); hit = ctx.block().and(I1, &hit, ¬_native_module); hit = ctx.block().and(I1, &hit, &token_match); hit = ctx.block().and(I1, &hit, &token_nonzero); - hit = ctx.block().and(I1, &hit, &slot_in_bounds); ctx.block().cond_br(&hit, &hit_label, &fallback_label); @@ -615,17 +595,13 @@ fn lower_put_value_static_write_ic( let cached2_slot_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "3")]); let slot2 = ctx.block().load(I64, &cached2_slot_ptr); let token2_match = ctx.block().icmp_eq(I64, &shape_token, &cached2_token); - let token2_nonzero = ctx.block().icmp_ne(I64, &shape_token, "0"); - let slot2_in_bounds = ctx.block().icmp_ult(I64, &slot2, &inline_limit); let mut hit2 = ctx.block().and(I1, &heap_candidate, &gc_object); hit2 = ctx.block().and(I1, &hit2, ¬_forwarded); hit2 = ctx.block().and(I1, &hit2, &flags_clear); - hit2 = ctx.block().and(I1, &hit2, ®ular); hit2 = ctx.block().and(I1, &hit2, &class_nonzero); hit2 = ctx.block().and(I1, &hit2, ¬_native_module); hit2 = ctx.block().and(I1, &hit2, &token2_match); - hit2 = ctx.block().and(I1, &hit2, &token2_nonzero); - hit2 = ctx.block().and(I1, &hit2, &slot2_in_bounds); + hit2 = ctx.block().and(I1, &hit2, &token_nonzero); ctx.block().cond_br(&hit2, &hit_label, &dispatch3_label); ctx.current_block = dispatch3_idx; @@ -639,17 +615,13 @@ fn lower_put_value_static_write_ic( let cached3_slot_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "5")]); let slot3 = ctx.block().load(I64, &cached3_slot_ptr); let token3_match = ctx.block().icmp_eq(I64, &shape_token, &cached3_token); - let token3_nonzero = ctx.block().icmp_ne(I64, &shape_token, "0"); - let slot3_in_bounds = ctx.block().icmp_ult(I64, &slot3, &inline_limit); let mut hit3 = ctx.block().and(I1, &heap_candidate, &gc_object); hit3 = ctx.block().and(I1, &hit3, ¬_forwarded); hit3 = ctx.block().and(I1, &hit3, &flags_clear); - hit3 = ctx.block().and(I1, &hit3, ®ular); hit3 = ctx.block().and(I1, &hit3, &class_nonzero); hit3 = ctx.block().and(I1, &hit3, ¬_native_module); hit3 = ctx.block().and(I1, &hit3, &token3_match); - hit3 = ctx.block().and(I1, &hit3, &token3_nonzero); - hit3 = ctx.block().and(I1, &hit3, &slot3_in_bounds); + hit3 = ctx.block().and(I1, &hit3, &token_nonzero); ctx.block().cond_br(&hit3, &hit_label, &dispatch4_label); ctx.current_block = dispatch4_idx; @@ -663,17 +635,13 @@ fn lower_put_value_static_write_ic( let cached4_slot_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "7")]); let slot4 = ctx.block().load(I64, &cached4_slot_ptr); let token4_match = ctx.block().icmp_eq(I64, &shape_token, &cached4_token); - let token4_nonzero = ctx.block().icmp_ne(I64, &shape_token, "0"); - let slot4_in_bounds = ctx.block().icmp_ult(I64, &slot4, &inline_limit); let mut hit4 = ctx.block().and(I1, &heap_candidate, &gc_object); hit4 = ctx.block().and(I1, &hit4, ¬_forwarded); hit4 = ctx.block().and(I1, &hit4, &flags_clear); - hit4 = ctx.block().and(I1, &hit4, ®ular); hit4 = ctx.block().and(I1, &hit4, &class_nonzero); hit4 = ctx.block().and(I1, &hit4, ¬_native_module); hit4 = ctx.block().and(I1, &hit4, &token4_match); - hit4 = ctx.block().and(I1, &hit4, &token4_nonzero); - hit4 = ctx.block().and(I1, &hit4, &slot4_in_bounds); + hit4 = ctx.block().and(I1, &hit4, &token_nonzero); ctx.block().cond_br(&hit4, &hit_label, &dispatch5_label); ctx.current_block = dispatch5_idx; @@ -867,7 +835,6 @@ fn lower_put_value_dyn_ic_inline( let ways_idx = ctx.new_block("put.dynic.ways"); let way1_idx = ctx.new_block("put.dynic.way1"); let way2_idx = ctx.new_block("put.dynic.way2"); - let bounds_idx = ctx.new_block("put.dynic.bounds"); let store_idx = ctx.new_block("put.dynic.store"); let slow_idx = ctx.new_block("put.dynic.slow"); let merge_idx = ctx.new_block("put.dynic.merge"); @@ -875,7 +842,6 @@ fn lower_put_value_dyn_ic_inline( let ways_label = ctx.block_label(ways_idx); let way1_label = ctx.block_label(way1_idx); let way2_label = ctx.block_label(way2_idx); - let bounds_label = ctx.block_label(bounds_idx); let store_label = ctx.block_label(store_idx); let slow_label = ctx.block_label(slow_idx); let merge_label = ctx.block_label(merge_idx); @@ -894,36 +860,31 @@ fn lower_put_value_dyn_ic_inline( let reserved_addr = ctx.block().sub(I64, &t_handle, "6"); let reserved_ptr = ctx.block().inttoptr(I64, &reserved_addr); let reserved = ctx.block().load(I16, &reserved_ptr); - let blocked = ctx.block().and(I16, &reserved, "6407"); // 0x1907 + let blocked = ctx + .block() + .and(I16, &reserved, &WRITE_PIC_BLOCKING_FLAGS.to_string()); let flags_clear = ctx.block().icmp_eq(I16, &blocked, "0"); - let object_type_ptr = ctx.block().inttoptr(I64, &t_handle); - let object_type = ctx.block().load(I32, &object_type_ptr); - let regular = ctx.block().icmp_eq(I32, &object_type, "1"); let class_addr = ctx.block().add(I64, &t_handle, "4"); let class_ptr = ctx.block().inttoptr(I64, &class_addr); let class_id = ctx.block().load(I32, &class_ptr); let class_nonzero = ctx.block().icmp_ne(I32, &class_id, "0"); let not_native_module = ctx.block().icmp_ne(I32, &class_id, "-2"); - let keys_addr = ctx.block().add(I64, &t_handle, "16"); - let keys_ptr = ctx.block().inttoptr(I64, &keys_addr); - let keys = ctx.block().load(I64, &keys_ptr); - let parent_class_addr = ctx.block().add(I64, &t_handle, "8"); - let parent_class_ptr = ctx.block().inttoptr(I64, &parent_class_addr); - let parent_class_id = ctx.block().load(I32, &parent_class_ptr); - let shape_id_rel = ctx.block().add(I32, &parent_class_id, "-2147483648"); + let shape_id_addr = ctx.block().add(I64, &t_handle, "8"); + let shape_id_ptr = ctx.block().inttoptr(I64, &shape_id_addr); + let raw_shape_id = ctx.block().load(I32, &shape_id_ptr); + let shape_id_rel = ctx.block().add(I32, &raw_shape_id, "-2147483648"); let has_shape_id = ctx.block().icmp_ult(I32, &shape_id_rel, "1073741824"); - let shape_id64 = ctx.block().zext(I32, &parent_class_id, I64); + let shape_id64 = ctx.block().zext(I32, &raw_shape_id, I64); let shape_id_token = ctx.block().or(I64, &shape_id64, "4611686018427387904"); let shape_token = ctx .block() - .select(I1, &has_shape_id, I64, &shape_id_token, &keys); + .select(I1, &has_shape_id, I64, &shape_id_token, "0"); let cached_token_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "0")]); let cached_token = ctx.block().load(I64, &cached_token_ptr); let token_match = ctx.block().icmp_eq(I64, &shape_token, &cached_token); let token_nonzero = ctx.block().icmp_ne(I64, &shape_token, "0"); let mut ok = ctx.block().and(I1, &gc_object, ¬_forwarded); ok = ctx.block().and(I1, &ok, &flags_clear); - ok = ctx.block().and(I1, &ok, ®ular); ok = ctx.block().and(I1, &ok, &class_nonzero); ok = ctx.block().and(I1, &ok, ¬_native_module); ok = ctx.block().and(I1, &ok, &token_match); @@ -936,42 +897,27 @@ fn lower_put_value_dyn_ic_inline( let s0_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "2")]); let s0 = ctx.block().load(I64, &s0_ptr); let hit0 = ctx.block().icmp_eq(I64, &k_bits, &k0); - ctx.block().cond_br(&hit0, &bounds_label, &way1_label); + ctx.block().cond_br(&hit0, &store_label, &way1_label); ctx.current_block = way1_idx; let k1_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "3")]); let k1 = ctx.block().load(I64, &k1_ptr); let s1_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "4")]); let s1 = ctx.block().load(I64, &s1_ptr); let hit1 = ctx.block().icmp_eq(I64, &k_bits, &k1); - ctx.block().cond_br(&hit1, &bounds_label, &way2_label); + ctx.block().cond_br(&hit1, &store_label, &way2_label); ctx.current_block = way2_idx; let k2_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "5")]); let k2 = ctx.block().load(I64, &k2_ptr); let s2_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "6")]); let s2 = ctx.block().load(I64, &s2_ptr); let hit2 = ctx.block().icmp_eq(I64, &k_bits, &k2); - ctx.block().cond_br(&hit2, &bounds_label, &slow_label); + ctx.block().cond_br(&hit2, &store_label, &slow_label); - ctx.current_block = bounds_idx; + ctx.current_block = store_idx; let slot = ctx.block().phi( I64, &[(&s0, &ways_label), (&s1, &way1_label), (&s2, &way2_label)], ); - let field_count_addr = ctx.block().add(I64, &t_handle, "12"); - let field_count_ptr = ctx.block().inttoptr(I64, &field_count_addr); - let field_count = ctx.block().load(I32, &field_count_ptr); - let field_count64 = ctx.block().zext(I32, &field_count, I64); - let below_floor = ctx - .block() - .icmp_ult(I64, &field_count64, INLINE_SLOT_FLOOR_LIT); - let inline_limit = - ctx.block() - .select(I1, &below_floor, I64, INLINE_SLOT_FLOOR_LIT, &field_count64); - let slot_in_bounds = ctx.block().icmp_ult(I64, &slot, &inline_limit); - ctx.block() - .cond_br(&slot_in_bounds, &store_label, &slow_label); - - ctx.current_block = store_idx; let header_words = (crate::target_layout::object_header_size_bytes(ctx.target_triple) / 8).to_string(); let slot_word = ctx.block().add(I64, &slot, &header_words); @@ -1005,12 +951,6 @@ fn lower_put_value_dyn_ic_inline( Ok(result) } -/// Inline-slot floor for emitted bounds checks — MUST match -/// perry-runtime `object::INLINE_SLOT_FLOOR` (the runtime pads every object -/// to at least this many physical slots; a codegen value larger than the -/// runtime's would widen inline stores into unallocated memory). -const INLINE_SLOT_FLOOR_LIT: &str = crate::target_layout::INLINE_SLOT_FLOOR_LIT; - fn static_write_key(ctx: &FnCtx<'_>, key: &Expr) -> Option { match key { Expr::String(property) => Some(property.clone()), diff --git a/crates/perry-codegen/src/lower_call/method_override.rs b/crates/perry-codegen/src/lower_call/method_override.rs index fda988e7b2..4c23e8078b 100644 --- a/crates/perry-codegen/src/lower_call/method_override.rs +++ b/crates/perry-codegen/src/lower_call/method_override.rs @@ -238,9 +238,8 @@ pub(super) fn emit_guarded_direct_method_call( .unwrap_or_else(|| crate::codegen::generic_method_body_name(direct_fn)); let expected_class_id_str = expected_class_id.to_string(); - let expected_keys_slot = - crate::expr::entry_init_load_rooted_global(ctx, &keys_global_name, I64); - let expected_keys = ctx.block().load(I64, &expected_keys_slot); + let expected_shape_id = + crate::typed_shape::load_class_shape_id(ctx, receiver_class_name, &keys_global_name); let key_idx = ctx.strings.intern(property); let entry = ctx.strings.entry(key_idx); @@ -258,14 +257,14 @@ pub(super) fn emit_guarded_direct_method_call( )) }; - // Per-arm keys tokens, loaded through the same entry-block init the - // declared class's token uses (module-init populates `@perry_class_keys_*` - // after the prelude, so the load may not be hoisted above it). - let subclass_keys: Vec = subclass_arms + // Per-arm ShapeIds, loaded through entry-block scalar slots. + let subclass_shape_ids: Vec = subclass_arms .iter() .map(|arm| { - let slot = crate::expr::entry_init_load_rooted_global(ctx, &arm.keys_global, I64); - ctx.block().load(I64, &slot) + let shape_global = + crate::typed_shape::shape_id_global_name_from_keys_global(&arm.keys_global); + let slot = ctx.func.entry_init_load_global(&shape_global, I32); + ctx.block().load(I32, &slot) }) .collect(); @@ -297,19 +296,19 @@ pub(super) fn emit_guarded_direct_method_call( // single-arm form keeps its original single call. let multi_arm = !subclass_arms.is_empty(); if multi_arm { - let keys_slot = ctx.func.alloca_entry(I64); + let shape_slot = ctx.func.alloca_entry(I32); let cid = ctx.block().call( I32, "js_method_direct_shape_class", - &[(DOUBLE, recv_box), (crate::types::PTR, &keys_slot)], + &[(DOUBLE, recv_box), (crate::types::PTR, &shape_slot)], ); - let keys = ctx.block().load(I64, &keys_slot); + let shape_id = ctx.block().load(I32, &shape_slot); { let next = sub_test_labels[0].clone(); let blk = ctx.block(); let cid_ok = blk.icmp_eq(I32, &cid, &expected_class_id_str); - let keys_ok = blk.icmp_eq(I64, &keys, &expected_keys); - let pass = blk.and(I1, &cid_ok, &keys_ok); + let shape_ok = blk.icmp_eq(I32, &shape_id, &expected_shape_id); + let pass = blk.and(I1, &cid_ok, &shape_ok); blk.cond_br(&pass, &fast_label, &next); } for (i, arm) in subclass_arms.iter().enumerate() { @@ -320,11 +319,11 @@ pub(super) fn emit_guarded_direct_method_call( .unwrap_or_else(|| fallback_label.clone()); let case_label = sub_case_labels[i].clone(); let class_id_str = arm.class_id.to_string(); - let arm_keys = subclass_keys[i].clone(); + let arm_shape_id = subclass_shape_ids[i].clone(); let blk = ctx.block(); let cid_ok = blk.icmp_eq(I32, &cid, &class_id_str); - let keys_ok = blk.icmp_eq(I64, &keys, &arm_keys); - let pass = blk.and(I1, &cid_ok, &keys_ok); + let shape_ok = blk.icmp_eq(I32, &shape_id, &arm_shape_id); + let pass = blk.and(I1, &cid_ok, &shape_ok); blk.cond_br(&pass, &case_label, &next); } ctx.current_block = guard_idx; @@ -340,7 +339,7 @@ pub(super) fn emit_guarded_direct_method_call( &[ (DOUBLE, recv_box), (I32, &expected_class_id_str), - (I64, &expected_keys), + (I32, &expected_shape_id), ], ) } else { @@ -354,7 +353,7 @@ pub(super) fn emit_guarded_direct_method_call( ), (DOUBLE, recv_box), (I32, &expected_class_id_str), - (I64, &expected_keys), + (I32, &expected_shape_id), (crate::types::PTR, &bytes_global), (I64, &name_len_str), (crate::types::PTR, &format!("@{}", direct_fn)), @@ -409,7 +408,7 @@ pub(super) fn emit_guarded_direct_method_call( (I64, &site_id), (DOUBLE, recv_box), (I32, &expected_class_id_str), - (I64, &expected_keys), + (I32, &expected_shape_id), (I64, &key_raw), (I32, &field_index_str), (I32, "1"), diff --git a/crates/perry-codegen/src/lower_call/mod.rs b/crates/perry-codegen/src/lower_call/mod.rs index d32ceeaac0..9e6d97e2b2 100644 --- a/crates/perry-codegen/src/lower_call/mod.rs +++ b/crates/perry-codegen/src/lower_call/mod.rs @@ -61,7 +61,7 @@ mod native; mod native_module_dispatch; mod native_table; mod new; -mod new_alloc; +pub(crate) mod new_alloc; mod new_ctor_args; mod new_helpers; mod omitted_native_params; diff --git a/crates/perry-codegen/src/lower_call/new_alloc.rs b/crates/perry-codegen/src/lower_call/new_alloc.rs index ced4293840..34e9bec7e8 100644 --- a/crates/perry-codegen/src/lower_call/new_alloc.rs +++ b/crates/perry-codegen/src/lower_call/new_alloc.rs @@ -19,31 +19,7 @@ use perry_hir::Class; use crate::expr::FnCtx; -use crate::types::{I1, I32, I64, I8, PTR}; - -/// Load the immutable ShapeId paired with a class's canonical keys global. -/// -/// As with `class_keys_slots`, cache it in a function-entry alloca: the inline -/// allocation slow path is an opaque runtime call, so LLVM will not reliably -/// hoist the module-global load out of a hot loop by itself. Unlike the keys -/// pointer this scalar is not a GC root and needs no shadow-slot binding. -pub(super) fn load_class_shape_id( - ctx: &mut FnCtx<'_>, - class_name: &str, - keys_global_name: &str, -) -> String { - let shape_slot = if let Some(slot) = ctx.class_shape_slots.get(class_name).cloned() { - slot - } else { - let shape_global = - crate::typed_shape::shape_id_global_name_from_keys_global(keys_global_name); - let slot = ctx.func.entry_init_load_global(&shape_global, I32); - ctx.class_shape_slots - .insert(class_name.to_string(), slot.clone()); - slot - }; - ctx.block().load(I32, &shape_slot) -} +use crate::types::{I32, I64, I8, PTR}; /// #7469: is the `new` site being lowered inside a **loop body**? /// @@ -380,7 +356,8 @@ fn emit_instance_alloc_inner( s }; let keys_ptr = ctx.block().load(I64, &keys_slot); - let shape_id = load_class_shape_id(ctx, class_name, &keys_global_name); + let shape_id = + crate::typed_shape::load_class_shape_id(ctx, class_name, &keys_global_name); ctx.pending_declares.push(( "js_object_alloc_class_inline_keys_stamped".to_string(), I64, @@ -501,7 +478,8 @@ fn emit_instance_alloc_inner( s }; let keys_ptr = ctx.block().load(I64, &keys_slot); - let shape_id = load_class_shape_id(ctx, class_name, &keys_global_name); + let shape_id = + crate::typed_shape::load_class_shape_id(ctx, class_name, &keys_global_name); // Inline bump-allocator IR. let blk = ctx.block(); @@ -588,14 +566,10 @@ fn emit_instance_alloc_inner( blk.store(I64, &oh_word_1.to_string(), &oh_addr_1); // Second 8 bytes: ShapeId (u32, low) | field_count (u32, high). - // Rung 0 removed the last inheritance consumer of this word; the - // parent edge was registered during module init. A zero id is the - // recoverable exhaustion path: retain the old parent word and let - // the still-authoritative pointer/count guards handle the object. + // The module-init runtime call either publishes a usable ShapeId + // or fail-stops on exhaustion; there is no pointer-token fallback. let oh_addr_2 = blk.gep(I8, &raw, &[(I64, "16")]); - let has_shape_id = blk.icmp_ne(I32, &shape_id, "0"); - let shape_word = blk.select(I1, &has_shape_id, I32, &shape_id, &parent_cid.to_string()); - let shape_word64 = blk.zext(I32, &shape_word, I64); + let shape_word64 = blk.zext(I32, &shape_id, I64); let oh_word_2 = blk.or( I64, &shape_word64, diff --git a/crates/perry-codegen/src/lower_call/property_get/dynamic_dispatch.rs b/crates/perry-codegen/src/lower_call/property_get/dynamic_dispatch.rs index 0844040beb..d70d779325 100644 --- a/crates/perry-codegen/src/lower_call/property_get/dynamic_dispatch.rs +++ b/crates/perry-codegen/src/lower_call/property_get/dynamic_dispatch.rs @@ -121,8 +121,10 @@ fn emit_tower_pshape_call( ) -> String { // The global is read ONCE per function (entry-hoisted); the case block only // reloads it from the stack slot, which mem2reg folds away. - let keys_slot = crate::expr::entry_init_load_rooted_global(ctx, &route.keys_global, I64); - let expected_keys = ctx.block().load(I64, &keys_slot); + let shape_global = + crate::typed_shape::shape_id_global_name_from_keys_global(&route.keys_global); + let shape_slot = ctx.func.entry_init_load_global(&shape_global, I32); + let expected_shape_id = ctx.block().load(I32, &shape_slot); let proven_idx = ctx.new_block(&format!("idispatch.case{}.pshape", case_no)); let generic_idx = ctx.new_block(&format!("idispatch.case{}.generic", case_no)); @@ -134,7 +136,7 @@ fn emit_tower_pshape_call( crate::expr::class_field_inline_guard::emit_proven_shape_recheck( ctx, recv_handle, - &expected_keys, + &expected_shape_id, &proven_label, &generic_label, ); diff --git a/crates/perry-codegen/src/lower_call/scalar_method.rs b/crates/perry-codegen/src/lower_call/scalar_method.rs index ab03768922..7629a8fbcc 100644 --- a/crates/perry-codegen/src/lower_call/scalar_method.rs +++ b/crates/perry-codegen/src/lower_call/scalar_method.rs @@ -605,7 +605,7 @@ fn materialize_scalar_receiver( slot }; let keys_ptr = ctx.block().load(I64, &keys_slot); - let shape_id = super::new_alloc::load_class_shape_id(ctx, class_name, &keys_global_name); + let shape_id = crate::typed_shape::load_class_shape_id(ctx, class_name, &keys_global_name); ctx.pending_declares.push(( "js_object_alloc_class_inline_keys_stamped".to_string(), I64, diff --git a/crates/perry-codegen/src/runtime_decls/objects.rs b/crates/perry-codegen/src/runtime_decls/objects.rs index fff879a568..1c490ba1b9 100644 --- a/crates/perry-codegen/src/runtime_decls/objects.rs +++ b/crates/perry-codegen/src/runtime_decls/objects.rs @@ -54,17 +54,6 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { // inline `header + 16 + idx*elem_size` load matches the runtime `data_ptr`). module.add_external_global("PERRY_TA_KIND_CACHE", "[64 x i64]"); module.add_external_global("PERRY_TA_VIEW_GUARD", I64); - // #6080a: process-global read-PIC epoch (perry-runtime - // `object::field_get_set::ic_miss::PERRY_IC_EPOCH`, starts at 1, bumped on - // every completed GC collection and at budgeted-sweep entry). The inline - // monomorphic property-get hit path compares its per-site `cache[2]` - // snapshot against this before trusting a raw keys-array POINTER token — - // the `@perry_ic_N` globals are invisible to every GC scanner, so a - // primed address that GC has since freed/moved would otherwise - // pointer-match a recycled keys array of a different shape and load the - // wrong slot. Shape-ID tokens (#6804, bit 62) skip the check: ids are - // never reused. - module.add_external_global("PERRY_IC_EPOCH", I64); module.declare_function("js_object_alloc", I64, &[I32, I32]); // #3149: `Object(value)` plain-call coercion. Takes & returns a NaN-boxed // JSValue (DOUBLE): nullish/primitive -> fresh {}, object passes through. @@ -163,7 +152,7 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { module.declare_function( "js_typed_feedback_class_field_set_guard", I32, - &[I64, DOUBLE, I32, I64, I64, I32, DOUBLE, I32], + &[I64, DOUBLE, I32, I32, I64, I32, DOUBLE, I32], ); // #5334 lever A: class-field-SET guard-MISS fallback, outlined. The cold arm // of the default diamond collapses from two calls (record_fallback + @@ -175,26 +164,26 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { ); // #5334 lever B: class-field-SET inline cache, FULLY outlined. For oversized // modules the whole diamond (guard + fast store + fallback) collapses to one - // call. Args: (site_id, recv, expected_class_id, expected_keys, key, + // call. Args: (site_id, recv, expected_class_id, expected_shape_id, key, // field_index, value, require_raw_f64). Same signature as the set guard. module.declare_function( "js_class_field_set_ic", VOID, - &[I64, DOUBLE, I32, I64, I64, I32, DOUBLE, I32], + &[I64, DOUBLE, I32, I32, I64, I32, DOUBLE, I32], ); module.declare_function( "js_typed_feedback_class_field_get_guard", I32, - &[I64, DOUBLE, I32, I64, I64, I32, I32], + &[I64, DOUBLE, I32, I32, I64, I32, I32], ); // #5391 path 2: class-field-GET inline cache, FULLY outlined. For oversized // modules the whole get diamond collapses to one call returning the field - // value. Args: (site_id, recv, expected_class_id, expected_keys, key, + // value. Args: (site_id, recv, expected_class_id, expected_shape_id, key, // field_index, require_raw_f64). Same signature as the get guard (+ f64 ret). module.declare_function( "js_class_field_get_ic", DOUBLE, - &[I64, DOUBLE, I32, I64, I64, I32, I32], + &[I64, DOUBLE, I32, I32, I64, I32, I32], ); module.declare_function( "js_typed_feedback_native_call_method", @@ -219,9 +208,9 @@ pub fn declare_phase_b_objects(module: &mut LlModule) { module.declare_function( "js_typed_feedback_method_direct_call_guard", I32, - &[I64, DOUBLE, I32, I64, PTR, I64, PTR], + &[I64, DOUBLE, I32, I32, PTR, I64, PTR], ); - module.declare_function("js_method_direct_shape_guard", I32, &[DOUBLE, I32, I64]); + module.declare_function("js_method_direct_shape_guard", I32, &[DOUBLE, I32, I32]); module.declare_function("js_method_direct_shape_class", I32, &[DOUBLE, PTR]); module.declare_function( "js_typed_feedback_closure_direct_call_guard", diff --git a/crates/perry-codegen/src/stmt/element_shape_loop.rs b/crates/perry-codegen/src/stmt/element_shape_loop.rs index 8e5521ca7e..eea6de96e4 100644 --- a/crates/perry-codegen/src/stmt/element_shape_loop.rs +++ b/crates/perry-codegen/src/stmt/element_shape_loop.rs @@ -866,7 +866,7 @@ pub(super) fn lower_element_shape_versioned_for( None => crate::expr::element_shape_guard::ElementShapeLoopTripCount::ArrayLength, }; let expected_class_id_str = matched.expected_class_id.to_string(); - let (elements_base, expected_keys, shape_ok, bound_i32) = + let (elements_base, expected_shape_id, shape_ok, bound_i32) = crate::expr::element_shape_guard::emit_element_shape_loop_preheader_check( ctx, matched.array_id, @@ -882,12 +882,6 @@ pub(super) fn lower_element_shape_versioned_for( // the clone is PROVEN call-free below. let deref_idx = ctx.current_block; - let max_field_index = matched - .fields - .values() - .copied() - .max() - .expect("matcher requires >= 1 tracked field"); let scope_id = ctx.next_loop_proof_scope_id(); let fast_scan_start = ctx.func.num_blocks(); ctx.current_block = fast_pre_idx; @@ -898,10 +892,9 @@ pub(super) fn lower_element_shape_versioned_for( scope_id, class_name: matched.class_name.clone(), elements_base, - expected_keys, + expected_shape_id, side_exit_label: slow_pre_label.clone(), fields: matched.fields.clone(), - max_field_index, element_binding: matched.element_binding, numeric_accumulator: matched.accumulator_id, }); diff --git a/crates/perry-codegen/src/stmt/loops.rs b/crates/perry-codegen/src/stmt/loops.rs index bd5a91e5be..e07d2c7e99 100644 --- a/crates/perry-codegen/src/stmt/loops.rs +++ b/crates/perry-codegen/src/stmt/loops.rs @@ -3839,19 +3839,17 @@ fn lower_class_field_versioned_for( // emitted IR is call-free, so the pointer the check validates is the // pointer the fast clone uses. let recv_box = lower_expr(ctx, &perry_hir::Expr::LocalGet(matched.recv_id))?; - let (obj_bits, obj_handle, expected_keys) = { + let expected_shape_id = crate::typed_shape::load_class_shape_id( + ctx, + &matched.class_name, + &matched.keys_global_name, + ); + let (obj_bits, obj_handle) = { let blk = ctx.block(); let obj_bits = blk.bitcast_double_to_i64(&recv_box); let obj_handle = blk.and(I64, &obj_bits, crate::nanbox::POINTER_MASK_I64); - let expected_keys = blk.load(I64, &format!("@{}", matched.keys_global_name)); - (obj_bits, obj_handle, expected_keys) + (obj_bits, obj_handle) }; - let max_field_index = matched - .fields - .values() - .map(|(field_index, _)| *field_index) - .max() - .expect("matcher requires >= 1 tracked field"); let has_store = matched.fields.values().any(|(_, written)| *written); let expected_class_id_str = matched.expected_class_id.to_string(); let (obj_ptr, shape_ok) = @@ -3860,8 +3858,7 @@ fn lower_class_field_versioned_for( &obj_bits, &obj_handle, &expected_class_id_str, - &expected_keys, - max_field_index, + &expected_shape_id, // Every tracked field is a raw-f64 candidate: reads rely on the // intact bit, so require it whether or not the loop stores. true, diff --git a/crates/perry-codegen/src/typed_shape.rs b/crates/perry-codegen/src/typed_shape.rs index bddd3b8f1d..7fd73ea465 100644 --- a/crates/perry-codegen/src/typed_shape.rs +++ b/crates/perry-codegen/src/typed_shape.rs @@ -383,3 +383,27 @@ pub(crate) fn shape_id_global_name_from_keys_global(keys_global_name: &str) -> S .map(|suffix| format!("perry_class_shape_id_{}", suffix)) .unwrap_or_else(|| format!("perry_class_shape_id_{}", keys_global_name)) } + +/// Load the immutable ShapeId paired with a class's canonical keys global. +/// +/// Cache it in a function-entry alloca: an opaque allocation/runtime call can +/// otherwise prevent LLVM from hoisting the module-global load out of a hot +/// loop. This scalar is not a GC root and needs no shadow-slot binding. +pub(crate) fn load_class_shape_id( + ctx: &mut crate::expr::FnCtx<'_>, + class_name: &str, + keys_global_name: &str, +) -> String { + let shape_slot = if let Some(slot) = ctx.class_shape_slots.get(class_name).cloned() { + slot + } else { + let shape_global = shape_id_global_name_from_keys_global(keys_global_name); + let slot = ctx + .func + .entry_init_load_global(&shape_global, crate::types::I32); + ctx.class_shape_slots + .insert(class_name.to_string(), slot.clone()); + slot + }; + ctx.block().load(crate::types::I32, &shape_slot) +} diff --git a/crates/perry-runtime/src/array/element_shape.rs b/crates/perry-runtime/src/array/element_shape.rs index ad5edbbd4a..35bfdc16ce 100644 --- a/crates/perry-runtime/src/array/element_shape.rs +++ b/crates/perry-runtime/src/array/element_shape.rs @@ -80,7 +80,7 @@ //! established at that recycled address next. //! //! Both follow the crate's convention for invalidation counters (`AtomicU64` -//! starting at 1, `PROP_PLAN_EPOCH` / `PERRY_IC_EPOCH`), not a per-thread +//! starting at 1, such as `PROP_PLAN_EPOCH`), not a per-thread //! `Cell`: the class registry a generation bump answers to is process-wide. //! //! ## Verified length: the structural half of the invalidation matrix @@ -232,12 +232,11 @@ pub(crate) fn invalidate_all_element_shapes() { /// The class id an element must have to keep the invariant, or `None` if the /// value cannot participate at all. /// -/// Strict on purpose. `POINTER_TAG` alone is not enough: `RegExpHeader` is -/// also tagged `GC_TYPE_OBJECT` (see the aliasing caution on `ObjectMeta`), -/// and reading `class_id` off a non-`ObjectHeader` payload yields garbage -/// that would then be *compared equal* across two unrelated arrays. -/// Requiring `object_type == OBJECT_TYPE_REGULAR` and a nonzero class id -/// keeps every accepted value a genuine shaped instance. +/// Strict on purpose. `POINTER_TAG` alone is not enough: every native heap +/// cell uses it, and reading `class_id` off a non-`ObjectHeader` payload yields +/// garbage that could compare equal across unrelated arrays. Requiring the +/// authoritative object kind/marker and a nonzero class id keeps every +/// accepted value a genuine shaped instance. #[inline] pub(crate) fn element_class_of_bits(value_bits: u64) -> Option { if value_bits & crate::value::TAG_MASK != crate::value::POINTER_TAG { @@ -256,7 +255,7 @@ pub(crate) fn element_class_of_bits(value_bits: u64) -> Option { return None; } let obj = addr as *const crate::object::ObjectHeader; - if (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR { + if !crate::object::object_is_regular(obj) { return None; } let class_id = (*obj).class_id; diff --git a/crates/perry-runtime/src/builtins/formatting.rs b/crates/perry-runtime/src/builtins/formatting.rs index a4cbc6c576..60de127878 100644 --- a/crates/perry-runtime/src/builtins/formatting.rs +++ b/crates/perry-runtime/src/builtins/formatting.rs @@ -851,9 +851,8 @@ pub(crate) fn format_jsvalue(value: f64, depth: usize) -> String { let buf_ptr = ptr as *const crate::buffer::BufferHeader; format_buffer_value(buf_ptr) } else if crate::regex::is_registered_regex(ptr as usize) { - // RegExp literals are GC_TYPE_OBJECT with no enumerable keys - // (generic formatter prints `{}`); render `/source/flags` - // instead (registry-gated, before the GC-header read; #800). + // RegExp literals have their own GC kind and no enumerable + // keys; render `/source/flags` instead (#800). collections::format_regexp(ptr as *const crate::regex::RegExpHeader) } else if crate::proxy::js_proxy_is_proxy(value) != 0 { format_proxy_value(value, depth, false) diff --git a/crates/perry-runtime/src/dgram.rs b/crates/perry-runtime/src/dgram.rs index d1d6409db5..0654d68a84 100644 --- a/crates/perry-runtime/src/dgram.rs +++ b/crates/perry-runtime/src/dgram.rs @@ -320,11 +320,12 @@ pub(crate) unsafe fn gc_type_for_ptr(raw: usize) -> Option { // `scripts/addr_class_inventory.py`'s `handle-floor` regex (it only // matches `ptr`/`addr`/`bits`-shaped identifiers) -- this site was debt // the ratchet could not even see. - if !crate::value::addr_class::is_plausible_heap_addr(raw) { - return None; - } - let header = (raw as *const u8).sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; - let gc_type = (*header).obj_type; + // A magnitude check alone does not prove that arbitrary receiver bits are + // mapped. Require arena or tracked-malloc ownership before reading the + // header, including for Linux AArch64 addresses in the upper low-48-bit + // userspace range (#8067). + let header = crate::value::addr_class::try_read_tracked_gc_header(raw)?; + let gc_type = (*header.as_ptr()).obj_type; if gc_type <= crate::gc::GC_TYPE_MAX { Some(gc_type) } else { @@ -772,4 +773,27 @@ mod gc_type_for_ptr_tests { ); } } + + #[test] + fn unmapped_heap_range_receiver_is_rejected_without_dereferencing() { + // This is inside the permissive heap-magnitude window but outside + // every Perry arena and tracked malloc allocation. A magnitude-only + // guard would read its unmapped predecessor as a GcHeader. + let addr = 0x0000_0001_0000_0000usize; + assert!(addr_class::is_plausible_heap_addr(addr)); + let boxed = f64::from_bits(JSValue::pointer(addr as *const u8).bits()); + assert!(object_ptr_from_value(boxed).is_none()); + } + + #[cfg(all( + target_arch = "aarch64", + any(target_os = "android", target_os = "linux") + ))] + #[test] + fn unmapped_upper_low_48_bit_receiver_is_rejected_without_dereferencing() { + let addr = 0x0000_e000_0000_1000usize; + assert!(addr_class::is_plausible_heap_addr(addr)); + let boxed = f64::from_bits(JSValue::pointer(addr as *const u8).bits()); + assert!(object_ptr_from_value(boxed).is_none()); + } } diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index 582e35150f..4f07489ea6 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -1675,15 +1675,6 @@ impl GcCycleState { fn step_sweep(&mut self, budget: GcWorkBudget) { let phase_start = trace_phase_start(&self.trace); if self.sweep_state.is_none() { - // #6080a: invalidate pointer-token read-PIC primes BEFORE the - // first address can be freed. A budgeted cycle's sweep slices - // interleave with the mutator, so waiting for the end-of-cycle - // `record_collection` bump would leave a window where a primed - // `@perry_ic_N` cache pointer-matches a keys array whose address - // an earlier slice already recycled. Primes taken after this - // bump reference marked (live-this-cycle) arrays, which later - // slices of this same sweep never free. - crate::object::pic_epoch_bump(); let full_trace = self.minor.is_none(); // Close the finalize->sweep gap: the barrier stayed enabled across // the mutator windows since AtomicFinalize ended. Trace whatever diff --git a/crates/perry-runtime/src/gc/heap_snapshot.rs b/crates/perry-runtime/src/gc/heap_snapshot.rs index d200962479..a662247855 100644 --- a/crates/perry-runtime/src/gc/heap_snapshot.rs +++ b/crates/perry-runtime/src/gc/heap_snapshot.rs @@ -207,7 +207,9 @@ unsafe fn object_field_name( obj: *const crate::object::ObjectHeader, field_index: usize, ) -> Option { - let keys_bits = (*obj).keys_array as u64; + let keys_bits = crate::object::shapes::object_shape_descriptor(obj) + .map(|descriptor| descriptor.keys) + .unwrap_or((*obj).keys_array as u64); let keys_addr = decode_slot_target(keys_bits); if keys_addr < GC_HEADER_SIZE + 0x1000 { return None; @@ -306,7 +308,11 @@ pub fn gc_build_v8_heap_snapshot_json() -> String { let mut ordinal: u32 = 0; let (fields_base, fields_len) = if rec.obj_type == GC_TYPE_OBJECT { let obj = rec.user as *const crate::object::ObjectHeader; - let fc = unsafe { (*obj).field_count } as usize; + let fc = unsafe { + crate::object::shapes::object_shape_descriptor(obj) + .map(|descriptor| descriptor.live_inline_slot_count as usize) + .unwrap_or((*obj).field_count as usize) + }; if fc <= 10_000 { ( rec.user + std::mem::size_of::(), diff --git a/crates/perry-runtime/src/gc/layout.rs b/crates/perry-runtime/src/gc/layout.rs index e227ca31e8..1de9345c0c 100644 --- a/crates/perry-runtime/src/gc/layout.rs +++ b/crates/perry-runtime/src/gc/layout.rs @@ -178,7 +178,10 @@ unsafe fn object_keys_array_ptr(user_ptr: usize) -> usize { if gc_type_layout_slot_kind((*header).obj_type) != GcLayoutSlotKind::ObjectFields { return 0; } - (*(user_ptr as *const crate::object::ObjectHeader)).keys_array as usize + let object = user_ptr as *const crate::object::ObjectHeader; + crate::object::shapes::object_shape_descriptor(object) + .map(|descriptor| descriptor.keys as usize) + .unwrap_or((*object).keys_array as usize) } /// Borrow the shared canonical descriptor for `user_ptr`'s shape, if @@ -198,13 +201,12 @@ unsafe fn with_shape_shared_descriptor( if keys == 0 { return None; } - // Defense-in-depth: the descriptor's `slot_count` is pinned to the owning - // object's `field_count` at install (`init_typed_shape_layout` rejects a - // mismatch). A differing current field_count means this object's shape is - // not the one the descriptor describes — e.g. a keys_array address reused by - // a shape with a different field count (moving-GC relocation before the new - // address is re-installed). Fall back (per-object → conservative). - let field_count = (*(user_ptr as *const crate::object::ObjectHeader)).field_count as usize; + // Defense-in-depth: both descriptor families must agree on the exact live + // bound. The ObjectHeader count is only an ABI mirror pending #8047. + let object = user_ptr as *const crate::object::ObjectHeader; + let field_count = crate::object::shapes::object_shape_descriptor(object) + .map(|descriptor| descriptor.live_inline_slot_count as usize) + .unwrap_or((*object).field_count as usize); let map = hot_shape_layouts().borrow(); let desc = map.get(&keys)?.as_ref()?; if desc.slot_count != field_count { @@ -411,7 +413,9 @@ pub(super) unsafe fn layout_header_for_user(user_ptr: usize) -> Option<*mut GcHe | GcLayoutSlotKind::ClosureCaptures => Some(header), // #6812: meta records keep no layout mask — their two child slots // (prototype, spill) are enumerated unconditionally. - GcLayoutSlotKind::None | GcLayoutSlotKind::ObjectMeta => None, + GcLayoutSlotKind::None | GcLayoutSlotKind::ObjectMeta | GcLayoutSlotKind::RegExpFields => { + None + } } } @@ -619,7 +623,10 @@ pub(crate) fn layout_note_slot(parent_user: usize, slot_index: usize, value_bits && (*header).obj_type == GC_TYPE_OBJECT { let object = parent_user as *const crate::object::ObjectHeader; - if slot_index < (*object).field_count as usize { + let live_slots = crate::object::shapes::object_shape_descriptor(object) + .map(|descriptor| descriptor.live_inline_slot_count as usize) + .unwrap_or((*object).field_count as usize); + if slot_index < live_slots { return; } } @@ -915,7 +922,10 @@ unsafe fn init_typed_shape_layout( return; } let obj_header = user_ptr as *const crate::object::ObjectHeader; - let object_slot_count = (*obj_header).field_count as usize; + let shape_descriptor = crate::object::shapes::object_shape_descriptor(obj_header); + let object_slot_count = shape_descriptor + .map(|descriptor| descriptor.live_inline_slot_count as usize) + .unwrap_or((*obj_header).field_count as usize); if object_slot_count != slot_count { layout_set_typed_unknown(header, user_ptr); return; @@ -965,7 +975,9 @@ unsafe fn init_typed_shape_layout( // `object_keys_array_ptr`'s two guards are already discharged above (the // low addresses were rejected, `GcLayoutSlotKind::ObjectFields` was // checked), so read the field directly rather than re-walking the header. - let keys = (*obj_header).keys_array as usize; + let keys = shape_descriptor + .map(|descriptor| descriptor.keys as usize) + .unwrap_or((*obj_header).keys_array as usize); let memo = if keys == 0 { None } else { @@ -1682,27 +1694,6 @@ pub(super) unsafe fn gc_child_slots(header: *mut GcHeader) -> HeapChildSlotItera .unwrap_or_else(HeapChildSlotIterator::empty) } GcLayoutSlotKind::ObjectFields => { - // Wall 18 follow-up: a `RegExpHeader` is allocated as - // `GC_TYPE_OBJECT` but is a NATIVE struct, NOT a shaped JS object. - // The generic ObjectHeader read takes `field_count` from offset 12, - // which for a `RegExpHeader` overlaps the high 32 bits of - // `pattern_ptr` (~900 on macOS's 0x3xx_… heap) → a bogus ~900-slot - // range that scans/rewrites ADJACENT heap during evacuation (heap - // corruption; `PERRY_GC_VERIFY_EVACUATION` reports it as a stale - // forwarded pointer "inside" the regex at an offset far past its - // size). This is a latent pre-existing bug — exposed deterministically - // once Wall 18 grew the header. Detect the regex via its - // self-identifying magic and scan EXACTLY its GC-visible slots — - // `pattern_ptr`/`flags_ptr` (a 2-slot contiguous payload range) and - // `last_index` (the prefix slot). The off-heap `regex_ptr`/`fancy_ptr`, - // the bool flags, the `magic` sentinel, and any tail padding are never - // inspected, so evacuation can never touch raw native data. - if crate::regex::regex_header_has_magic(user_ptr as *const crate::regex::RegExpHeader) { - let (pattern_slot, slot_count, last_index_slot) = - crate::regex::regex_gc_slot_ptrs(user_ptr as *mut crate::regex::RegExpHeader); - let range = HeapSlotRange::new(pattern_slot, slot_count); - return HeapChildSlotIterator::new(header, Some(last_index_slot), range); - } let obj = user_ptr as *mut crate::object::ObjectHeader; let Some(range) = crate::object::gc_field_slot_range(obj) else { return HeapChildSlotIterator::empty(); @@ -1717,6 +1708,15 @@ pub(super) unsafe fn gc_child_slots(header: *mut GcHeader) -> HeapChildSlotItera HeapChildSlotIterator::new(header, keys_slot, range) .with_meta_slot(crate::object::gc_object_meta_slot(user_ptr as usize)) } + GcLayoutSlotKind::RegExpFields => { + let (pattern_slot, slot_count, last_index_slot) = + crate::regex::regex_gc_slot_ptrs(user_ptr as *mut crate::regex::RegExpHeader); + HeapChildSlotIterator::new( + header, + Some(last_index_slot), + HeapSlotRange::new(pattern_slot, slot_count), + ) + } GcLayoutSlotKind::ObjectMeta => { // #6812: prototype (NaN-boxed / raw / sentinel) as the prefix // slot, the raw spill-buffer pointer as a 1-slot range. Mirrors diff --git a/crates/perry-runtime/src/gc/layout_slot_visit.rs b/crates/perry-runtime/src/gc/layout_slot_visit.rs index 388b802de3..fa917ca842 100644 --- a/crates/perry-runtime/src/gc/layout_slot_visit.rs +++ b/crates/perry-runtime/src/gc/layout_slot_visit.rs @@ -15,39 +15,44 @@ pub(super) unsafe fn visit_gc_layout_slot_descriptors( visit: &mut dyn FnMut(GcMutableSlotDescriptor), ) { let mut child_slots = gc_child_slots(header); - // Capture the authoritative pre-visit facts. A copying visit can rewrite - // `keys_array`, and a sibling may already have rewritten the shared - // descriptor, so the descriptor helper accepts exactly the old OR new - // pointer — never an unrelated pointer that merely shares an id. + // Capture the authoritative descriptor facts. `gc_child_slots` has already + // copied the descriptor's keys edge into the compatibility header scratch + // slot; a copying visit can rewrite that slot, after which the descriptor + // table is updated below. let object_shape_facts = if (*header).obj_type == GC_TYPE_OBJECT { let obj = (header as *mut u8).add(GC_HEADER_SIZE) as *mut crate::object::ObjectHeader; - if crate::regex::regex_header_has_magic(obj as *const crate::regex::RegExpHeader) { - None + let descriptor = crate::object::shapes::object_shape_descriptor(obj); + let old_keys = descriptor + .map(|facts| facts.keys as usize as *mut crate::array::ArrayHeader) + .unwrap_or((*obj).keys_array); + let live_inline_slot_count = descriptor + .map(|facts| facts.live_inline_slot_count) + .unwrap_or((*obj).field_count); + if old_keys.is_null() { + Some((obj, 0, 0, live_inline_slot_count)) + } else if crate::value::addr_class::try_read_tracked_gc_header(old_keys as usize) + .is_some_and(|keys_header| (*keys_header.as_ptr()).obj_type == GC_TYPE_ARRAY) + { + // A forwarded tracked array still carries GC_TYPE_ARRAY in + // its from-space header. The length helper follows that stub, + // so a sibling whose shared keys edge was already rewritten + // can still validate against the descriptor's new pointer. + Some(( + obj, + old_keys as u64, + descriptor + .map(|facts| facts.logical_key_count) + .unwrap_or_else(|| { + crate::array::keys_array_len_capped_to_capacity(old_keys) as u32 + }), + live_inline_slot_count, + )) } else { - let old_keys = (*obj).keys_array; - let live_inline_slot_count = (*obj).field_count; - if old_keys.is_null() { - Some((obj, 0, 0, live_inline_slot_count)) - } else if crate::value::addr_class::try_read_tracked_gc_header(old_keys as usize) - .is_some_and(|keys_header| (*keys_header.as_ptr()).obj_type == GC_TYPE_ARRAY) - { - // A forwarded tracked array still carries GC_TYPE_ARRAY in - // its from-space header. The length helper follows that stub, - // so a sibling whose shared keys edge was already rewritten - // can still validate against the descriptor's new pointer. - Some(( - obj, - old_keys as u64, - crate::array::keys_array_len_capped_to_capacity(old_keys) as u32, - live_inline_slot_count, - )) - } else { - // Do not dereference corrupt/unmapped header words merely - // because their sibling word happens to look like a ShapeId. - // The authoritative header edge below is still enumerated; - // only redundant descriptor synchronization is skipped. - None - } + // Do not dereference corrupt/unmapped header words merely because + // their sibling word happens to look like a ShapeId. The + // authoritative header edge below is still enumerated; only + // redundant descriptor synchronization is skipped. + None } } else { None @@ -61,8 +66,8 @@ pub(super) unsafe fn visit_gc_layout_slot_descriptors( // work may retain enumerated slot addresses across budgeted resumptions, // during which descriptor insertion can reallocate the table. A deferred // visitor leaves old==new here; the metadata forwarding pass repairs it - // after copying. RegExp aliases GC_TYPE_OBJECT with a different native - // header and was excluded while capturing the facts above. + // after copying. RegExp uses its dedicated GC slot kind and never enters + // the ObjectHeader branch above. if let Some((obj, old_keys, logical_key_count, live_inline_slot_count)) = object_shape_facts { let new_keys = (*obj).keys_array as u64; // Mark, verify, and deferred dirty scans leave the header edge @@ -176,6 +181,9 @@ pub(super) unsafe fn visit_gc_rewrite_slot_descriptors( }, ); } + GcRewriteDescriptorKind::RegExp => { + visit_gc_layout_slot_descriptors(header, &mut visit); + } GcRewriteDescriptorKind::Closure => { visit_gc_layout_slot_descriptors(header, &mut visit); crate::closure::visit_closure_dynamic_prop_value_slots_mut(user_ptr as usize, |slot| { diff --git a/crates/perry-runtime/src/gc/roots/runtime_handles.rs b/crates/perry-runtime/src/gc/roots/runtime_handles.rs index b2e5036200..1137cb3228 100644 --- a/crates/perry-runtime/src/gc/roots/runtime_handles.rs +++ b/crates/perry-runtime/src/gc/roots/runtime_handles.rs @@ -204,6 +204,27 @@ impl<'scope> RuntimeHandle<'scope> { }) } + /// Pass the handle's current mutable pointer to `f` without exposing a + /// bare handle read at the call site. + /// + /// This is the argument-position companion to [`Self::across_mut`]. Use it + /// when a rooted pointer must be handed directly to a non-allocating + /// operation or to a runtime entry point that establishes its own root + /// before it can allocate. The callback must not retain the pointer: this + /// method scopes the raw value, but it cannot keep that value current if a + /// collection moves the allocation while `f` is running. Use + /// [`Self::across_mut`] when the caller needs a post-collection address. + #[inline] + pub fn with_mut_ptr(&self, f: impl FnOnce(*mut T) -> R) -> R { + f(self.get_raw_mut_ptr::()) + } + + /// `with_mut_ptr` for a `*const` argument. See its safety contract. + #[inline] + pub fn with_const_ptr(&self, f: impl FnOnce(*const T) -> R) -> R { + f(self.get_raw_const_ptr::()) + } + /// Run `f` — which may allocate, and therefore may MOVE the object this /// handle roots — and return its result together with the object's /// **post-collection** address. diff --git a/crates/perry-runtime/src/gc/telemetry.rs b/crates/perry-runtime/src/gc/telemetry.rs index ce610ce5da..79d91c6a0e 100644 --- a/crates/perry-runtime/src/gc/telemetry.rs +++ b/crates/perry-runtime/src/gc/telemetry.rs @@ -48,15 +48,7 @@ impl GcStats { /// recent-pause ring advance together with the counters, so no future /// collection path can update one without the others. /// - /// #6080a: the read-PIC epoch bump rides the same funnel — every - /// completed collection may have freed or moved a keys array whose raw - /// address is primed in a `@perry_ic_N` cache no GC scanner can see, so - /// pointer-token primes must stop hitting from here on. (Budgeted cycles - /// bump a second time at sweep ENTRY — see `step_sweep` — because their - /// sweep slices interleave with the mutator before this funnel runs. - /// Double-bumping is harmless: it only costs one extra re-prime.) pub(super) fn record_collection(&mut self, freed_bytes: u64, elapsed_us: u64) { - crate::object::pic_epoch_bump(); self.collection_count += 1; self.total_freed_bytes = self.total_freed_bytes.saturating_add(freed_bytes); self.last_pause_us = elapsed_us; diff --git a/crates/perry-runtime/src/gc/tests/copying.rs b/crates/perry-runtime/src/gc/tests/copying.rs index ce343809cf..78b2ee2a07 100644 --- a/crates/perry-runtime/src/gc/tests/copying.rs +++ b/crates/perry-runtime/src/gc/tests/copying.rs @@ -1058,6 +1058,27 @@ fn root_source_runtime_handle_rewrite_is_attributed_to_runtime_handles() { assert!(trace.root_sources.runtime_handles.rewritten_slots > 0); } +#[test] +fn with_pointer_callbacks_receive_the_current_handle_slot_address() { + let _guard = CopyingNurseryTestGuard::new(0); + let _trigger_guard = GcTriggerThresholdTestGuard::suppress_automatic_triggers(); + gc_register_mutable_root_scanner_with_source( + scan_runtime_handle_roots_mut, + MutableRootScannerSource::RuntimeHandles, + ); + let child = young_leaf(); + let scope = RuntimeHandleScope::new(); + let handle = scope.root_raw_mut_ptr(child as *mut u8); + + let trace = collect_minor_trace(GcTriggerKind::Direct); + let fresh_mut = handle.with_mut_ptr::(|ptr| ptr as usize); + let fresh_const = handle.with_const_ptr::(|ptr| ptr as usize); + + assert_copied_minor_trace(&trace, true, CopiedMinorFallbackReason::None, false); + assert_ne!(fresh_mut, child, "the callback received the stale address"); + assert_eq!(fresh_const, fresh_mut, "mutable/const callbacks disagree"); +} + #[test] fn across_mut_hands_back_the_post_collection_address() { // #7341 layer 3: the whole point of `across_mut` is that the pre-call diff --git a/crates/perry-runtime/src/gc/tests/copying/survival_and_malloc.rs b/crates/perry-runtime/src/gc/tests/copying/survival_and_malloc.rs index 0be9461c6a..83130cf246 100644 --- a/crates/perry-runtime/src/gc/tests/copying/survival_and_malloc.rs +++ b/crates/perry-runtime/src/gc/tests/copying/survival_and_malloc.rs @@ -789,6 +789,45 @@ fn test_movable_date_evacuation_migrates_expando_and_preserves_ts() { ); } +#[test] +fn test_movable_regexp_evacuation_migrates_all_address_owned_state() { + assert!(crate::gc::gc_type_is_movable(crate::gc::GC_TYPE_REGEXP)); + + let _guard = CopyingNurseryTestGuard::new(1); + let re = crate::regex::test_alloc_nursery_regexp_for_move("move/source", "gi"); + let old_addr = re as usize; + assert!(crate::arena::pointer_in_nursery(old_addr)); + assert!(crate::regex::test_regex_pointer_entry_exists(old_addr)); + assert!(crate::regex::test_regex_source_entry_exists(old_addr)); + + crate::object::exotic_expando::test_seed_exotic_expando_entry( + old_addr, + "tag", + crate::value::JSValue::int32(42).bits(), + ); + js_shadow_slot_set(0, ptr_bits(old_addr)); + + let _ = gc_collect_minor(); + + let new_addr = (js_shadow_slot_get(0) & POINTER_MASK) as usize; + assert_ne!(new_addr, 0, "rooted RegExp must survive the copied minor"); + assert_ne!(new_addr, old_addr, "the RegExp must be evacuated"); + assert!(crate::regex::regex_header_has_magic(new_addr as *const _)); + + assert!(crate::regex::test_regex_pointer_entry_exists(new_addr)); + assert!(!crate::regex::test_regex_pointer_entry_exists(old_addr)); + assert!(crate::regex::test_regex_source_entry_exists(new_addr)); + assert!(!crate::regex::test_regex_source_entry_exists(old_addr)); + assert!(crate::object::exotic_expando::test_exotic_expando_entry_exists(new_addr)); + assert!(!crate::object::exotic_expando::test_exotic_expando_entry_exists(old_addr)); + + let source = crate::regex::js_regexp_get_source(new_addr as *const _); + assert_eq!(crate::regex::string_as_str(source), r"move\/source"); + let reloaded_addr = (js_shadow_slot_get(0) & POINTER_MASK) as usize; + let flags = crate::regex::js_regexp_get_flags(reloaded_addr as *const _); + assert_eq!(crate::regex::string_as_str(flags), "gi"); +} + // #6181: the promotion-handoff census switched from the unfiltered // `arena_walk_objects_with_block_index` (visits every object in every region, // discards out-of-range ones in the callback) to the block-filtered walk that diff --git a/crates/perry-runtime/src/gc/tests/layout_trace/per_object_tables.rs b/crates/perry-runtime/src/gc/tests/layout_trace/per_object_tables.rs index a1fd1dafc4..bbf06adf7d 100644 --- a/crates/perry-runtime/src/gc/tests/layout_trace/per_object_tables.rs +++ b/crates/perry-runtime/src/gc/tests/layout_trace/per_object_tables.rs @@ -409,7 +409,10 @@ fn test_immortal_scope_stores_trace_without_taking_a_side_table_entry() { let child = crate::object::js_object_alloc(0, 0); let child_header = unsafe { header_from_user_ptr(child as *const u8) }; unsafe { - *(obj as *mut u8).add(8).cast::().add(1) = POINTER_TAG | (child as u64 & POINTER_MASK); + let fields = (obj as *mut u8) + .add(std::mem::size_of::()) + .cast::(); + *fields.add(1) = POINTER_TAG | (child as u64 & POINTER_MASK); } { let _immortal = ImmortalLayoutScope::new(); diff --git a/crates/perry-runtime/src/gc/tests/mod.rs b/crates/perry-runtime/src/gc/tests/mod.rs index 43041e9d95..0459e9e790 100644 --- a/crates/perry-runtime/src/gc/tests/mod.rs +++ b/crates/perry-runtime/src/gc/tests/mod.rs @@ -42,6 +42,7 @@ mod runtime_roots; mod scan_fallback; mod schedule; mod shadow_stack_ops; +mod shape_descriptor_authority; mod smoke; mod step_bounds; pub(super) mod support; diff --git a/crates/perry-runtime/src/gc/tests/shape_descriptor_authority.rs b/crates/perry-runtime/src/gc/tests/shape_descriptor_authority.rs new file mode 100644 index 0000000000..5b11c4746e --- /dev/null +++ b/crates/perry-runtime/src/gc/tests/shape_descriptor_authority.rs @@ -0,0 +1,60 @@ +use super::super::*; + +#[test] +fn gc_recovers_keys_and_live_slots_from_shape_id_after_header_sabotage() { + let _lock = global_side_table_test_lock(); + unsafe { + let scope = crate::gc::RuntimeHandleScope::new(); + let obj_handle = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 2)); + let key_a = crate::string::js_string_from_bytes(b"shape_gc_a".as_ptr(), 10); + let key_a_handle = scope.root_string_ptr(key_a); + let key_b = crate::string::js_string_from_bytes(b"shape_gc_b".as_ptr(), 10); + let key_b_handle = scope.root_string_ptr(key_b); + obj_handle.with_mut_ptr::(|obj| { + key_a_handle.with_mut_ptr::(|key_a| { + crate::object::js_object_set_field_by_name( + obj, + key_a, + crate::value::js_nanbox_pointer(key_a as i64), + ); + }); + }); + obj_handle.with_mut_ptr::(|obj| { + key_b_handle.with_mut_ptr::(|key_b| { + crate::object::js_object_set_field_by_name( + obj, + key_b, + crate::value::js_nanbox_pointer(key_b as i64), + ); + }); + }); + + obj_handle.with_mut_ptr::(|obj| { + let descriptor = crate::object::shapes::object_shape_descriptor(obj) + .expect("published object must have an authoritative descriptor"); + assert_eq!(descriptor.logical_key_count, 2); + assert_eq!(descriptor.live_inline_slot_count, 2); + + // These are ABI mirrors until #8047. Corrupt both to prove the GC walk + // derives its strong keys edge and exact payload range from ShapeId. + // GC_STORE_AUDIT(POINTER_FREE): test sabotage removes the compatibility edge by storing null. + (*obj).keys_array = std::ptr::null_mut(); + (*obj).field_count = 0; + + let slots = super::support::test_heap_child_slots_for_user(obj as *mut u8); + assert_eq!((*obj).keys_array as u64, descriptor.keys); + + let fields = + (obj as *mut u8).add(std::mem::size_of::()) as *mut u64; + let child_slots: Vec<*mut u64> = slots + .into_iter() + .filter_map(|slot| match slot { + HeapChildSlot::Child(ptr, _) => Some(ptr), + HeapChildSlot::PointerFreeRange(_) => None, + }) + .collect(); + assert!(child_slots.contains(&fields)); + assert!(child_slots.contains(&fields.add(1))); + }); + } +} diff --git a/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs b/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs index 4955255691..6c26de77e4 100644 --- a/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs +++ b/crates/perry-runtime/src/gc/tests/telemetry_verifier.rs @@ -577,28 +577,3 @@ fn test_pause_ring_records_max_and_window() { assert!(recent_max >= GC_RECENT_PAUSE_WINDOW as u64); assert!(recent_avg > 0 && recent_avg <= recent_max); } - -/// #6080a: `record_collection` is the single per-collection funnel, and the -/// read-PIC epoch bump rides it — a completed collection may have recycled a -/// keys-array address some `@perry_ic_N` cache still holds as a raw pointer -/// token, so every collection must strand those primes. Asserting >= (not ==) -/// keeps the test robust against concurrent collections on other test threads -/// (the epoch is process-global by design). -#[test] -fn test_record_collection_bumps_read_pic_epoch() { - use std::sync::atomic::Ordering; - let before = crate::object::PERRY_IC_EPOCH.load(Ordering::Relaxed); - assert!( - before >= 1, - "epoch starts at 1 so zeroinitializer never hits" - ); - GC_STATS.with(|stats| { - stats.borrow_mut().record_collection(0, 1); - }); - let after = crate::object::PERRY_IC_EPOCH.load(Ordering::Relaxed); - assert!( - after > before, - "every completed collection must advance PERRY_IC_EPOCH \ - (before={before}, after={after})" - ); -} diff --git a/crates/perry-runtime/src/gc/types.rs b/crates/perry-runtime/src/gc/types.rs index 8e1b1de86e..dadbe0a7dd 100644 --- a/crates/perry-runtime/src/gc/types.rs +++ b/crates/perry-runtime/src/gc/types.rs @@ -61,7 +61,11 @@ pub const GC_TYPE_TEMPORAL: u8 = 18; /// its owner's header slot, so ordinary tracing gives it exactly the /// owner's lifetime; movable, and holds one traced NaN-box slot. pub const GC_TYPE_OBJECT_META: u8 = 19; -pub const GC_TYPE_MAX: u8 = GC_TYPE_OBJECT_META; +/// Native `RegExpHeader`. RegExp used to share `GC_TYPE_OBJECT`, forcing every +/// ObjectHeader consumer to inspect unrelated payload words for a magic value. +/// A distinct GC kind is the authoritative, header-external discriminator. +pub const GC_TYPE_REGEXP: u8 = 20; +pub const GC_TYPE_MAX: u8 = GC_TYPE_REGEXP; pub(super) const MALLOC_KIND_UNKNOWN_INDEX: usize = 0; pub(super) const MALLOC_KIND_BUCKET_COUNT: usize = GC_TYPE_MAX as usize + 1; @@ -153,6 +157,7 @@ pub(crate) enum GcRewriteDescriptorKind { Leaf, Array, Object, + RegExp, Closure, Promise, Error, @@ -171,6 +176,7 @@ pub(crate) enum GcLayoutSlotKind { None, ArrayElements, ObjectFields, + RegExpFields, ClosureCaptures, /// #6812: ObjectMeta records carry two live edges — the custom /// `[[Prototype]]` value and the raw spill-buffer pointer. Before the @@ -215,6 +221,10 @@ pub(crate) enum GcMoveHookKind { /// move. Errors are movable; without this a moved error lost its /// `err.code`/`err.syscall`/user-assigned props. ErrorSideTables, + /// Rekey RegExp identity/source registries plus its exotic expando owner + /// entry. `GC_TYPE_REGEXP` is movable, and all three tables use the + /// payload address as their key. + RegExpSideTables, } #[allow(dead_code)] @@ -258,6 +268,11 @@ pub(crate) enum GcFinalizeHookKind { /// #7539: free a dead lazy JSON array's tape bytes, which /// `json_tape_store` owns outside the GC heap. LazyArrayTape, + /// Drop a dead RegExp cell's entries from every payload-address-keyed + /// registry. Arena reclamation reaches the equivalent cleanup through the + /// move-hook dead-owner fan-out; malloc-tracked cells use this finalize + /// hook instead. + RegExpSideTables, } #[allow(dead_code)] @@ -650,6 +665,21 @@ pub(super) static GC_TYPE_INFO_BY_ID: [Option; MALLOC_KIND_BUCKET_CO GcRewriteHookKind::None, GcFinalizeHookKind::None, )), + Some(gc_type_info_entry( + GC_TYPE_REGEXP, + "regexp", + GcAllocationPolicy::ArenaOrMalloc, + true, + GcRewriteDescriptorKind::RegExp, + GcLayoutSlotKind::RegExpFields, + true, + GcExternalBytePolicy::InlinePayload, + GcLargeObjectPolicy::MallocTracked, + false, + GcMoveHookKind::RegExpSideTables, + GcRewriteHookKind::None, + GcFinalizeHookKind::RegExpSideTables, + )), ]; #[inline] @@ -757,6 +787,9 @@ pub(crate) fn gc_type_after_payload_move(obj_type: u8, old_user: usize, new_user old_user, new_user, ); } + GcMoveHookKind::RegExpSideTables => { + crate::regex::regex_header_moved_for_gc(old_user, new_user); + } } } @@ -780,6 +813,9 @@ pub(crate) fn gc_type_clear_dead_payload_side_tables(obj_type: u8, user_ptr: usi GcMoveHookKind::ErrorSideTables => { crate::node_submodules::diagnostics_gc::error_side_tables_clear_dead(user_ptr); } + GcMoveHookKind::RegExpSideTables => { + crate::regex::regex_header_clear_dead_for_gc(user_ptr); + } GcMoveHookKind::None | GcMoveHookKind::MapSideTables | GcMoveHookKind::SetSideTables @@ -835,6 +871,9 @@ pub(crate) unsafe fn gc_type_finalize_unmarked_payload(obj_type: u8, user_ptr: * GcFinalizeHookKind::LazyArrayTape => { crate::json_tape_store::release(user_ptr as usize); } + GcFinalizeHookKind::RegExpSideTables => { + crate::regex::regex_header_clear_dead_for_gc(user_ptr as usize); + } } } @@ -873,6 +912,11 @@ pub(crate) fn validate_gc_type_info(info: &GcTypeInfo) -> Result<(), &'static st return Err("object rewrite descriptor must expose object field slots"); } } + GcRewriteDescriptorKind::RegExp => { + if info.layout_slot_kind != GcLayoutSlotKind::RegExpFields { + return Err("regexp rewrite descriptor must expose regexp fields"); + } + } GcRewriteDescriptorKind::Closure => { if info.layout_slot_kind != GcLayoutSlotKind::ClosureCaptures { return Err("closure rewrite descriptor must expose closure capture slots"); @@ -1044,6 +1088,12 @@ pub const OBJ_FLAG_ARRAY_DESCRIPTORS: u16 = 0x400; // path is always correct). #7480 reuses bit 11 for `GC_TYPE_ARRAY` as // `GC_ARRAY_ELEMENT_SHAPE`; the two are disjoint by `obj_type`. pub const OBJ_FLAG_HAS_DESCRIPTORS: u16 = 0x800; +/// Heap class-expression value (`class C {}`), as distinct from an ordinary +/// instance carrying the same `GC_TYPE_OBJECT` allocation tag. This is the +/// authoritative replacement for `ObjectHeader::object_type == +/// OBJECT_TYPE_CLASS`; the legacy payload word remains an ABI mirror until +/// #8047 removes it. Bit 13 is preserved by survival-age and layout-state +/// updates and is otherwise unused for `GC_TYPE_OBJECT`. // #2145: this object is a per-kind `.prototype` whose // `[[Prototype]]` is the shared `%TypedArray%.prototype` intrinsic. // `Object.getPrototypeOf(Int8Array.prototype)` returns the cached diff --git a/crates/perry-runtime/src/native_abi.rs b/crates/perry-runtime/src/native_abi.rs index 5db307c76c..ae042884ad 100644 --- a/crates/perry-runtime/src/native_abi.rs +++ b/crates/perry-runtime/src/native_abi.rs @@ -351,7 +351,7 @@ pub extern "C" fn js_native_abi_check_pod_object(value: f64) -> i64 { let gc_header = (obj as *const u8).sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; let is_gc_object = (*gc_header).obj_type == crate::gc::GC_TYPE_OBJECT; - let is_regular = (*obj).object_type == crate::error::OBJECT_TYPE_REGULAR; + let is_regular = crate::object::object_is_regular(obj); if !is_gc_object || !is_regular { throw_type_error("Expected object for native pod parameter"); } diff --git a/crates/perry-runtime/src/object/alloc.rs b/crates/perry-runtime/src/object/alloc.rs index 9045d2228b..074210bda1 100644 --- a/crates/perry-runtime/src/object/alloc.rs +++ b/crates/perry-runtime/src/object/alloc.rs @@ -159,6 +159,7 @@ pub extern "C" fn js_object_alloc_with_parent( ptr::write(fields_ptr.add(i), JSValue::undefined()); } crate::gc::layout_init_pointer_free(ptr as *mut u8); + crate::object::shapes::synchronize_object_shape_descriptor(ptr); ptr } @@ -187,6 +188,7 @@ pub extern "C" fn js_object_alloc_fast(class_id: u32, field_count: u32) -> *mut // GC_STORE_AUDIT(INIT): freshly allocated object starts with no keys-array edge. (*ptr).keys_array = ptr::null_mut(); crate::gc::layout_init_pointer_free(ptr as *mut u8); + crate::object::shapes::synchronize_object_shape_descriptor(ptr); } ptr @@ -221,6 +223,7 @@ pub extern "C" fn js_object_alloc_fast_with_parent( // GC_STORE_AUDIT(INIT): freshly allocated object starts with no keys-array edge. (*ptr).keys_array = ptr::null_mut(); crate::gc::layout_init_pointer_free(ptr as *mut u8); + crate::object::shapes::synchronize_object_shape_descriptor(ptr); } ptr @@ -314,14 +317,17 @@ pub extern "C" fn js_object_alloc_class_inline_keys( ) -> *mut ObjectHeader { let ptr = object_alloc_class_inline_keys_impl(class_id, parent_class_id, field_count, keys_array); - if !keys_array.is_null() { - unsafe { - let id = crate::object::shapes::shape_id_for_keys_ensure( - keys_array as *const ArrayHeader, - (*keys_array).length, - ); - crate::object::shapes::birth_stamp_object_shape(ptr, id); - } + unsafe { + let key_count = if keys_array.is_null() { + 0 + } else { + (*keys_array).length + }; + let id = crate::object::shapes::shape_id_for_keys_ensure( + keys_array as *const ArrayHeader, + key_count, + ); + crate::object::shapes::birth_stamp_object_shape(ptr, id); } ptr } @@ -332,8 +338,8 @@ pub extern "C" fn js_object_alloc_class_inline_keys( /// initialization. Installing it after the existing allocator returns keeps /// every allocation/rooting/layout invariant above in one implementation, /// while making a fresh class instance immediately usable by ShapeId guards. -/// A zero/exhausted id preserves the allocation-time parent word; the retained -/// pointer/count guards remain the fail-closed source of truth. +/// ShapeId exhaustion fail-stops during module initialization; no newborn can +/// be published with a pointer/count fallback identity. #[no_mangle] pub extern "C" fn js_object_alloc_class_inline_keys_stamped( class_id: u32, @@ -1572,15 +1578,9 @@ pub unsafe extern "C" fn js_object_assign_one(target_f64: f64, source_f64: f64) }; let source_is_array = source_obj_type == crate::gc::GC_TYPE_ARRAY; - // #7341: a RegExp source must be skipped here, and the exotic guard above - // cannot do it. That guard classifies by GC type, and a RegExp is literally - // `gc_malloc(GC_TYPE_OBJECT)` (see `regex.rs`) — so unlike Map/Set/Date it - // passes `== GC_TYPE_OBJECT` and falls into the plain-object arm, where - // `(*src).keys_array` reads a `RegExpHeader` at `ObjectHeader`'s field - // offset. That is a type confusion: the slot it lands on is not a keys - // array, and `js_array_length` then reads a GcHeader at `garbage - 8`. - // Under from-space quarantine that address is a retired protected page and - // the process dies; unprotected it silently walks unrelated memory. + // #7341: a RegExp source has no ObjectHeader keys array and must not enter + // the plain-object copy arm. Its dedicated GC kind makes that decision + // without reading any native payload word. // // Per CopyDataProperties a RegExp exposes no own enumerable string keys // through this path (`source`/`flags`/`lastIndex` are prototype accessors @@ -1588,10 +1588,9 @@ pub unsafe extern "C" fn js_object_assign_one(target_f64: f64, source_f64: f64) // `Object.assign({}, /x/g)` is `{}`. Any own expandos a user attached live // in the exotic-expando side table, which this raw walk never read anyway. // - // `is_regex_pointer` is the bounds-checked magic probe, safe on arbitrary - // payloads. Repro: `Object.assign({}, /x/g)` under + // Repro: `Object.assign({}, /x/g)` under // PERRY_GC_PROTECT_FROMSPACE=1 PERRY_GC_HEAP_LIMIT=8. - if crate::regex::is_regex_pointer(src_raw as *const u8) { + if source_obj_type == crate::gc::GC_TYPE_REGEXP { return target_f64; } diff --git a/crates/perry-runtime/src/object/class_registry/parent_static.rs b/crates/perry-runtime/src/object/class_registry/parent_static.rs index 177c1eb050..7d2e136432 100644 --- a/crates/perry-runtime/src/object/class_registry/parent_static.rs +++ b/crates/perry-runtime/src/object/class_registry/parent_static.rs @@ -418,15 +418,28 @@ pub extern "C" fn js_get_dynamic_parent_value(class_id: u32) -> f64 { } /// #1789: stamp a freshly-allocated object as a heap "class object" (the -/// value a class EXPRESSION evaluates to). Sets `object_type = -/// OBJECT_TYPE_CLASS` so `typeof` reports "function" and `new`/`instanceof` -/// read `class_id` from it. Called by codegen right after `js_object_alloc` -/// in the `ClassExprFresh` lowering. +/// value a class EXPRESSION evaluates to). Transitions the authoritative +/// ShapeId descriptor kind and updates `object_type` only as a compatibility +/// mirror. Called by codegen right after `js_object_alloc` in the +/// `ClassExprFresh` lowering. #[no_mangle] pub extern "C" fn js_object_mark_class(obj: i64) { if obj != 0 { unsafe { + let Some(header) = crate::value::addr_class::try_read_gc_header(obj as usize) else { + return; + }; + if header.obj_type != crate::gc::GC_TYPE_OBJECT + || header.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 + { + return; + } + // Compatibility mirror only; all semantic reads use the ShapeId + // descriptor kind so #8047 can remove this payload word atomically. (*(obj as *mut ObjectHeader)).object_type = crate::error::OBJECT_TYPE_CLASS; + // Becoming a class object changes dispatch semantics even though + // the rooted keys and slot layout stay the same. + crate::object::shapes::transition_object_shape_to_class(obj as *mut ObjectHeader); // #6530: record cid → class object so `instance.constructor` // resolves to the SAME value the module scope/exports hold (see // `CLASS_OBJECT_VALUES`). The template cid was stamped by the @@ -437,33 +450,20 @@ pub extern "C" fn js_object_mark_class(obj: i64) { } } -/// #1789: is `ptr` a heap "class object" (`object_type == OBJECT_TYPE_CLASS`)? -/// Validates the GcHeader is a `GC_TYPE_OBJECT` before reading `object_type`, +/// #1789: is `ptr` a heap class object? +/// Validates the GcHeader is a live `GC_TYPE_OBJECT` before reading its ShapeId, /// so raw Map/Set/Buffer pointers (no GcHeader) are never misread. Used by /// `typeof`, `new`, and `instanceof` to recognize a class value. pub fn is_class_object_ptr(ptr: *const u8) -> bool { - // Reject anything in the native-module handle band (see - // `value::addr_class`). Those are registry ids (net.Socket, zlib stream, - // crypto, fastify, ioredis, timers, …) bit-OR'd with POINTER_TAG, not real - // heap pointers — real objects always live above the band. The previous - // 0x1008 floor only caught the tiny net/fastify id space; a mid-range - // handle (e.g. zlib's stream base, #1843) sailed past it and this function - // then segfaulted dereferencing `[handle - 8]` as a GcHeader. - if crate::value::addr_class::is_handle_band(ptr as usize) { - return false; - } - // #5226: small typed arrays and `Buffer`s (incl. `new Uint8Array(n)`, which - // lowers to a slab-allocated Buffer) are off-GC-heap with no GcHeader, so - // the `ptr - GC_HEADER_SIZE` back-read below faults when the block sits at - // the start of a freshly mapped region. They are never class objects — - // reject via the side tables first (no back-read). - if crate::typedarray::is_offheap_sidetable_alloc(ptr as usize) { - return false; - } unsafe { - let gc_header = ptr.sub(crate::gc::GC_HEADER_SIZE) as *const crate::gc::GcHeader; - (*gc_header).obj_type == crate::gc::GC_TYPE_OBJECT - && (*(ptr as *const ObjectHeader)).object_type == crate::error::OBJECT_TYPE_CLASS + let Some(header) = crate::value::addr_class::try_read_gc_header(ptr as usize) else { + return false; + }; + header.obj_type == crate::gc::GC_TYPE_OBJECT + && header.gc_flags & crate::gc::GC_FLAG_FORWARDED == 0 + && crate::object::shapes::object_shape_descriptor(ptr.cast()).is_some_and(|shape| { + shape.object_kind == crate::object::shapes::ShapeObjectKind::Class + }) } } @@ -1710,3 +1710,224 @@ pub fn method_owner_class_id(class_id: u32, name: &str) -> Option { } None } + +#[cfg(test)] +mod shape_authority_tests_8067 { + fn key<'scope>( + scope: &'scope crate::gc::RuntimeHandleScope, + name: &str, + ) -> crate::gc::RuntimeHandle<'scope> { + scope.root_string_ptr(crate::string::js_string_from_bytes( + name.as_ptr(), + name.len() as u32, + )) + } + + #[test] + fn mark_class_rejects_non_heap_addresses() { + // Representative ids from the native-handle and proxy bands. The + // extern entry point must validate before reading a preceding header. + super::js_object_mark_class(0x40000); + super::js_object_mark_class(1); + } + + #[test] + fn saved_class_lineage_beats_an_interim_shape_self_heal() { + let _lock = crate::gc::global_side_table_test_lock(); + unsafe { + const CID: u32 = 0x8068; + let scope = crate::gc::RuntimeHandleScope::new(); + let obj_handle = scope.root_raw_mut_ptr(crate::object::js_object_alloc(CID, 1)); + let ((), obj) = obj_handle.across_mut::(|| { + obj_handle.with_mut_ptr::(|obj| { + super::js_object_mark_class(obj as i64) + }) + }); + let predecessor = crate::object::shapes::object_shape_descriptor(obj) + .expect("marked class descriptor"); + assert_eq!( + predecessor.object_kind, + crate::object::shapes::ShapeObjectKind::Class + ); + + // Model a re-entrant shape observer in the narrow mutation window: + // the structural mutator has saved its predecessor and cleared the + // stamp, then typed feedback defensively self-heals the object. + assert!(crate::object::shapes::clear_object_shape_stamp(obj)); + let (interim, obj) = obj_handle.across_mut::(|| { + crate::typed_feedback::test_object_shape_token(obj as usize) + }); + assert_eq!( + crate::object::shapes::shape_descriptor_by_id(interim as u32) + .expect("interim descriptor") + .object_kind, + crate::object::shapes::ShapeObjectKind::Ordinary, + "test premise: a lineage-free self-heal is ordinary" + ); + + crate::object::shapes::synchronize_object_shape_descriptor_from(obj, Some(predecessor)); + assert_eq!( + crate::object::shapes::object_shape_descriptor(obj) + .expect("restored descriptor") + .object_kind, + crate::object::shapes::ShapeObjectKind::Class, + "the mutator's saved semantic lineage must outrank an interim self-heal" + ); + } + } + + #[test] + fn class_kind_survives_static_field_installation_and_deletion() { + let _lock = crate::gc::global_side_table_test_lock(); + unsafe { + const CID: u32 = 0x8067; + let scope = crate::gc::RuntimeHandleScope::new(); + let obj_handle = scope.root_raw_mut_ptr(crate::object::js_object_alloc(CID, 8)); + let before = obj_handle.with_mut_ptr::(|obj| { + let before = crate::object::shapes::object_shape_id(obj); + assert!(crate::object::object_is_regular(obj)); + before + }); + + let ((), obj) = obj_handle.across_mut::(|| { + obj_handle.with_mut_ptr::(|obj| { + super::js_object_mark_class(obj as i64) + }) + }); + let after = crate::object::shapes::object_shape_id(obj); + assert_ne!(before, after, "becoming a class object is semantic"); + assert_eq!( + crate::object::shapes::object_shape_descriptor(obj) + .expect("class descriptor") + .object_kind, + crate::object::shapes::ShapeObjectKind::Class + ); + + // Sabotage the compatibility mirror. Classification must remain + // driven by the ShapeId descriptor transition above. + (*obj).object_type = crate::error::OBJECT_TYPE_REGULAR; + assert!(super::is_class_object_ptr(obj.cast())); + assert!(!crate::object::object_is_regular(obj)); + + // Numeric layout installation historically set/cleared bits in + // GcHeader::_reserved, where the old class marker collided with + // GC_LAYOUT_ALL_POINTERS. Shape kind must be unaffected. + let numeric_key = key(&scope, "numericStatic"); + let ((), obj) = obj_handle.across_mut::(|| { + obj_handle.with_mut_ptr::(|obj| { + numeric_key.with_const_ptr::(|key| { + crate::object::js_object_set_field_by_name(obj, key, 42.0) + }) + }) + }); + assert!( + super::is_class_object_ptr(obj.cast()), + "numeric static write changed class descriptor: {:?}", + crate::object::shapes::object_shape_descriptor(obj) + ); + assert!(!crate::object::object_is_regular(obj)); + + // Repeat with a pointer-bearing static value, which drives the + // opposite GC layout state and used to erase the aliased bit. + let pointer_key = key(&scope, "pointerStatic"); + let payload = key(&scope, "rootedStaticValue"); + let ((), obj) = obj_handle.across_mut::(|| { + obj_handle.with_mut_ptr::(|obj| { + pointer_key.with_const_ptr::(|key| { + payload.with_mut_ptr::(|payload| { + let value = + f64::from_bits(crate::value::JSValue::string_ptr(payload).bits()); + crate::object::js_object_set_field_by_name(obj, key, value) + }) + }) + }) + }); + assert!(super::is_class_object_ptr(obj.cast())); + assert!(!crate::object::object_is_regular(obj)); + assert_eq!( + crate::object::shapes::object_shape_descriptor(obj) + .expect("post-write class descriptor") + .object_kind, + crate::object::shapes::ShapeObjectKind::Class, + "class kind must never share storage with GC layout flags" + ); + + // The pointer-bearing write above leaves a typed/side-mask layout. + // Growing the keys array from that state invalidates typed + // feedback. The invalidation asks for the receiver's shape, so a + // keys transition must keep the old class stamp visible until the + // invalidation finishes and must prefer its saved predecessor over + // any defensive self-heal in the temporary cleared-stamp window. + let after_pointer_key = key(&scope, "afterPointerStatic"); + let ((), obj) = obj_handle.across_mut::(|| { + obj_handle.with_mut_ptr::(|obj| { + after_pointer_key.with_const_ptr::(|key| { + crate::object::js_object_set_field_by_name(obj, key, 7.0) + }) + }) + }); + assert!( + super::is_class_object_ptr(obj.cast()), + "typed-layout invalidation erased class descriptor lineage: {:?}", + crate::object::shapes::object_shape_descriptor(obj) + ); + assert_eq!( + crate::object::shapes::object_shape_descriptor(obj) + .expect("post-invalidation class descriptor") + .object_kind, + crate::object::shapes::ShapeObjectKind::Class + ); + + // Deletion installs a cloned keys array, which clears the current + // stamp. The replacement descriptor must inherit class kind from + // the predecessor captured before that clear. + let (deleted, obj) = obj_handle.across_mut::(|| { + obj_handle.with_mut_ptr::(|obj| { + numeric_key.with_const_ptr::(|key| { + crate::object::js_object_delete_field(obj, key) + }) + }) + }); + assert_eq!(deleted, 1); + assert!( + super::is_class_object_ptr(obj.cast()), + "deleting a static field erased class descriptor lineage: {:?}", + crate::object::shapes::object_shape_descriptor(obj) + ); + assert_eq!( + crate::object::shapes::object_shape_descriptor(obj) + .expect("post-delete class descriptor") + .object_kind, + crate::object::shapes::ShapeObjectKind::Class + ); + + let class_value = crate::value::js_nanbox_pointer(obj as i64); + let class_value_handle = scope.root_nanbox_f64(class_value); + let typeof_ptr = crate::builtins::js_value_typeof(class_value); + assert_eq!(crate::regex::string_as_str(typeof_ptr), "function"); + + let instance = crate::object::js_new_function_construct( + class_value_handle.get_nanbox_f64(), + std::ptr::null(), + 0, + ); + let instance_handle = scope.root_nanbox_f64(instance); + let instance_value = crate::value::JSValue::from_bits(instance.to_bits()); + assert!( + instance_value.is_pointer(), + "construction must return an object" + ); + let instance_ptr = instance_value.as_pointer::(); + assert_eq!((*instance_ptr).class_id, CID); + assert_eq!( + crate::object::js_instanceof_dynamic( + instance_handle.get_nanbox_f64(), + class_value_handle.get_nanbox_f64(), + ) + .to_bits(), + crate::value::TAG_TRUE, + "instanceof must still recognize the class after static writes" + ); + } + } +} diff --git a/crates/perry-runtime/src/object/delete_rest.rs b/crates/perry-runtime/src/object/delete_rest.rs index d214f8c88c..2f8443f4d9 100644 --- a/crates/perry-runtime/src/object/delete_rest.rs +++ b/crates/perry-runtime/src/object/delete_rest.rs @@ -320,6 +320,10 @@ pub extern "C" fn js_object_delete_field( } (*keys_cloned).length = new_count as u32; super::rebuild_array_layout_from_slots(keys_cloned); + // Preserve semantic generation and object kind before installing the + // cloned keys array: `set_object_keys_array` clears the old stamp when + // it observes the pointer change. + let predecessor = crate::object::shapes::object_shape_descriptor(obj); set_object_keys_array(obj, keys_cloned); // 1) Shift values down: for slot j in i..new_count, copy slot j+1 @@ -408,7 +412,7 @@ pub extern "C" fn js_object_delete_field( // `shape_slot_lookup`'s shrink check already anticipates), so // deleting it would silently make that path wrong. crate::object::shapes::clear_object_shape_stamp(obj); - crate::object::shapes::synchronize_object_shape_descriptor(obj); + crate::object::shapes::synchronize_object_shape_descriptor_from(obj, predecessor); 1 } diff --git a/crates/perry-runtime/src/object/descriptor_state.rs b/crates/perry-runtime/src/object/descriptor_state.rs index 94f422af67..436c804b48 100644 --- a/crates/perry-runtime/src/object/descriptor_state.rs +++ b/crates/perry-runtime/src/object/descriptor_state.rs @@ -426,6 +426,10 @@ pub(crate) fn note_descriptor_target(obj: usize) { if header.obj_type == crate::gc::GC_TYPE_OBJECT { let header = header as *const crate::gc::GcHeader as *mut crate::gc::GcHeader; (*header)._reserved |= crate::gc::OBJ_FLAG_HAS_DESCRIPTORS; + let object = obj as *mut crate::object::ObjectHeader; + if crate::object::object_is_shaped(object) { + crate::object::shapes::transition_object_shape_semantics(object); + } } } } @@ -689,12 +693,22 @@ pub(crate) fn set_property_attrs(obj: usize, key: String, attrs: PropertyAttrs) /// Remove a customized property descriptor for (obj, key), restoring default /// data-property attributes for subsequent writes and reflection. pub(crate) fn clear_property_attrs(obj: usize, key: &str) { - super::prop_plan::prop_plan_epoch_bump(); - state() + let removed = state() .descriptors .property_descriptors .borrow_mut() - .remove(&(obj, key.to_string())); + .remove(&(obj, key.to_string())) + .is_some(); + if !removed { + return; + } + super::prop_plan::prop_plan_epoch_bump(); + unsafe { + let object = obj as *mut crate::object::ObjectHeader; + if crate::object::object_is_shaped(object) { + crate::object::shapes::transition_object_shape_semantics(object); + } + } } /// Look up the accessor descriptor (get/set) for (obj, key). @@ -875,12 +889,22 @@ pub(crate) fn set_accessor_descriptor(obj: usize, key: String, acc: AccessorDesc /// Remove an accessor descriptor for (obj, key), letting ordinary data-property /// reads and writes use the object's stored field again. pub(crate) fn clear_accessor_descriptor(obj: usize, key: &str) { - super::prop_plan::prop_plan_epoch_bump(); - state() + let removed = state() .descriptors .accessor_descriptors .borrow_mut() - .remove(&(obj, key.to_string())); + .remove(&(obj, key.to_string())) + .is_some(); + if !removed { + return; + } + super::prop_plan::prop_plan_epoch_bump(); + unsafe { + let object = obj as *mut crate::object::ObjectHeader; + if crate::object::object_is_shaped(object) { + crate::object::shapes::transition_object_shape_semantics(object); + } + } } /// Install a built-in *reflection-only* accessor descriptor for (obj, key) diff --git a/crates/perry-runtime/src/object/exotic_expando.rs b/crates/perry-runtime/src/object/exotic_expando.rs index 73076681c1..26f76abb96 100644 --- a/crates/perry-runtime/src/object/exotic_expando.rs +++ b/crates/perry-runtime/src/object/exotic_expando.rs @@ -16,10 +16,10 @@ //! closures piggyback on the generic side tables (`PROPERTY_DESCRIPTORS` / //! `ACCESSOR_DESCRIPTORS`), which are already keyed by raw address. //! -//! GC: all three cell kinds are non-movable, so the address key is stable -//! for the cell's lifetime. Stored values are kept alive via a registered -//! mutable root scanner. Address reuse after a sweep is handled by clearing -//! the table slot at allocation time (`expando_clear_on_alloc`). +//! GC: address keys are migrated by each movable owner's registered move +//! hook. Stored values are kept alive via a mutable root scanner. Address +//! reuse after a sweep is handled by clearing the table slot at allocation +//! time (`expando_clear_on_alloc`). use std::cell::{Cell, RefCell}; use std::collections::HashMap; @@ -65,8 +65,8 @@ pub(crate) enum ExoticKind { } /// Classify `addr` as a Date cell, RegExp header, or Error header. Returns -/// `None` for everything else (including the small-handle band). One -/// `GcHeader` read; the RegExp set probe only runs for `GC_TYPE_OBJECT`. +/// `None` for everything else (including the small-handle band). Every arm is +/// selected directly by its authoritative `GcHeader` kind. pub(crate) fn exotic_expando_kind(addr: usize) -> Option { let gc = unsafe { crate::value::addr_class::try_read_gc_header(addr) }?; match gc.obj_type { @@ -76,9 +76,7 @@ pub(crate) fn exotic_expando_kind(addr: usize) -> Option { crate::gc::GC_TYPE_PROMISE => Some(ExoticKind::Promise), crate::gc::GC_TYPE_MAP => Some(ExoticKind::Map), crate::gc::GC_TYPE_SET => Some(ExoticKind::Set), - crate::gc::GC_TYPE_OBJECT if crate::regex::is_regex_pointer(addr as *const u8) => { - Some(ExoticKind::RegExp) - } + crate::gc::GC_TYPE_REGEXP => Some(ExoticKind::RegExp), _ => None, } } @@ -224,15 +222,21 @@ pub(crate) fn expando_clear_on_alloc(addr: usize) { tables.entries.borrow_mut().remove(&addr); } +/// Drop an expando entry when its owner is finalized directly rather than +/// discovered by the shared dead-owner pruning pass. +pub(crate) fn exotic_expando_owner_clear_dead(addr: usize) { + expando_clear_on_alloc(addr); +} + /// Death pruning (2026-07-09 GC audit wave 2): the root scanner /// (`scan_exotic_expando_roots_mut`) strongly roots EVERY owner's values, /// dead owners included, so a dead Date/RegExp/Promise/Map/Set's expando /// value graph was immortal until the exact address happened to be handed /// to a new cell of the same kind (`expando_clear_on_alloc`). Prune entries /// whose owner cell is provably dead instead. `is_dead_owner` is one of the -/// GC's deadness predicates (`gc::dead_owner`). Note: non-movable Date / -/// Temporal cells that die PINNED are skipped by the predicate's pinned -/// check and remain covered by the clear-on-alloc path. +/// GC's deadness predicates (`gc::dead_owner`). Owners that die PINNED are +/// skipped by the predicate's pinned check and remain covered by the +/// clear-on-alloc path. pub(crate) fn prune_dead_exotic_expando_owners(is_dead_owner: &dyn Fn(usize) -> bool) { let tables = &crate::state::state().exotic_expando; if !tables.in_use.get() { @@ -266,12 +270,12 @@ pub(crate) fn test_exotic_expando_entry_exists(addr: usize) -> bool { } /// Rekey a movable exotic cell's expando entry after the GC relocates it from -/// `old_addr` to `new_addr`. Date / RegExp / Temporal cells are non-movable so -/// this never fires for them, but a `Promise` (`GC_TYPE_PROMISE`) is movable — -/// without this, a `.then()`-chained thenable that survives a GC move would -/// lose the `status`/`value` expandos it was gated on. Stored expando *values* -/// are already rewritten by `scan_exotic_expando_roots_mut`; this migrates the -/// owner *key*. Wired via `GcMoveHookKind::ExoticExpandoOwner`. +/// `old_addr` to `new_addr`. Without this, a surviving owner would lose its +/// user-defined properties after a move. Stored expando *values* are already +/// rewritten by `scan_exotic_expando_roots_mut`; this migrates the owner +/// *key*. Most users wire this directly via +/// `GcMoveHookKind::ExoticExpandoOwner`; RegExp calls it from its combined +/// side-table move hook. pub(crate) fn exotic_expando_owner_moved(old_addr: usize, new_addr: usize) { let tables = &crate::state::state().exotic_expando; if !tables.in_use.get() || old_addr == new_addr { diff --git a/crates/perry-runtime/src/object/field_get_set.rs b/crates/perry-runtime/src/object/field_get_set.rs index bee6ff4f5b..f832cff995 100644 --- a/crates/perry-runtime/src/object/field_get_set.rs +++ b/crates/perry-runtime/src/object/field_get_set.rs @@ -330,7 +330,6 @@ pub(crate) use has_property::{ wide_key_index_note_hit, WIDE_KEY_INDEX_MIN_KEYS, }; pub use has_property::{js_in_operator, js_object_has_property}; -pub(crate) use ic_miss::pic_epoch_bump; pub(crate) use ic_miss::{ is_array_method_value_name, is_primitive_proto_method, is_timer_handle_method_key, set_method_value_name, @@ -338,7 +337,7 @@ pub(crate) use ic_miss::{ pub use ic_miss::{ js_object_get_field_by_name_f64, js_object_get_field_by_property_id_f64, js_object_get_field_ic_miss, js_object_set_field_by_property_id, js_private_brand_check, - js_private_guard, PicCache, PERRY_IC_EPOCH, PIC_CACHE_WORDS, + js_private_guard, PicCache, PIC_CACHE_WORDS, }; #[cfg(test)] diff --git a/crates/perry-runtime/src/object/field_get_set/field_ops.rs b/crates/perry-runtime/src/object/field_get_set/field_ops.rs index c11fc9a012..e82c912915 100644 --- a/crates/perry-runtime/src/object/field_get_set/field_ops.rs +++ b/crates/perry-runtime/src/object/field_get_set/field_ops.rs @@ -193,20 +193,16 @@ pub extern "C" fn js_object_get_class_id(obj: *const ObjectHeader) -> u32 { return 0; } let addr = obj as usize; - // Built-in headers (Set / Map / Regex) live in their own per-type + // Built-in headers (Set / Map) live in their own per-type // registries — they're never user class instances. Reject them first. // // The reason given here used to be that Set/Map headers are `std::alloc`'d // with no `GcHeader` at `obj - 8`. That stopped being true when // `js_set_alloc` / `js_map_alloc` moved to // `arena_alloc_gc(_, _, GC_TYPE_SET|GC_TYPE_MAP)` — both DO carry a header, - // and the `GC_TYPE_OBJECT` test below already rejects them on it. Regex - // pointers are the remaining header-less case, which is why the registry - // order is kept. - if crate::set::is_registered_set(addr) - || crate::map::is_registered_map(addr) - || crate::regex::is_regex_pointer(obj as *const u8) - { + // and the GC-kind test below already rejects them on it. RegExp likewise + // has a dedicated kind, so it needs no payload or registry discriminator. + if crate::set::is_registered_set(addr) || crate::map::is_registered_map(addr) { return 0; } unsafe { diff --git a/crates/perry-runtime/src/object/field_get_set/get_field_by_name_tail.rs b/crates/perry-runtime/src/object/field_get_set/get_field_by_name_tail.rs index ef9ae3bb9a..312dd23dc8 100644 --- a/crates/perry-runtime/src/object/field_get_set/get_field_by_name_tail.rs +++ b/crates/perry-runtime/src/object/field_get_set/get_field_by_name_tail.rs @@ -1034,14 +1034,11 @@ pub(crate) fn get_field_by_name_object_tail( gc_type == crate::gc::GC_TYPE_MAP, ); } - // RegExp: RegExpHeader is allocated via GC_TYPE_OBJECT but tracked - // in REGEX_POINTERS. Detect and route `.source`, `.flags`, + // RegExp has a dedicated GC kind. Route `.source`, `.flags`, // `.lastIndex`, `.global`, `.ignoreCase`, `.multiline`, `.sticky`, - // `.unicode`, `.dotAll` to the regex header fields. Must run - // before the generic object-field path so the keys_array lookup - // doesn't try to read the regex header bytes as ObjectHeader. - if gc_type == crate::gc::GC_TYPE_OBJECT && crate::regex::is_regex_pointer(obj as *const u8) - { + // `.unicode`, `.dotAll` to the regex header fields. The kind check keeps + // its native payload out of the generic ObjectHeader field path. + if gc_type == crate::gc::GC_TYPE_REGEXP { if !key.is_null() { let key_ptr = (key as *const u8).add(std::mem::size_of::()); let key_len = (*key).byte_len as usize; @@ -1139,10 +1136,7 @@ pub(crate) fn get_field_by_name_object_tail( return JSValue::undefined(); } if gc_type != crate::gc::GC_TYPE_OBJECT { - let object_type = (*obj).object_type; - if object_type != crate::error::OBJECT_TYPE_REGULAR { - return JSValue::undefined(); - } + return JSValue::undefined(); } if super::super::is_arguments_object(obj) { if let Some(value) = super::super::arguments_object_get_field(obj, key) { @@ -1287,7 +1281,7 @@ pub(crate) fn get_field_by_name_object_tail( let key_bytes = std::slice::from_raw_parts(key_ptr, key_len); // #4949 `.prototype` / #6497 `.name` on heap class-expression // values — see `class_object_props`. - if (*obj).object_type == crate::error::OBJECT_TYPE_CLASS && (*obj).class_id != 0 { + if super::super::is_class_object_ptr(obj as *const u8) && (*obj).class_id != 0 { if key_bytes == b"prototype" { return super::class_object_props::class_object_prototype_value(obj); } diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs index a3f2d2a279..152b823aeb 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss.rs @@ -185,52 +185,6 @@ pub(crate) fn is_timer_handle_method_key(key: &[u8]) -> bool { ) } -/// #6759 C3c: is `keys` safe to prime into a per-site PIC cache whose hit -/// path does an UNVALIDATED compare-and-load? True only for -/// `GC_FLAG_SHAPE_SHARED` arrays — those are shape-cache-resident -/// (process-rooted, so they stay LIVE for as long as a cache references -/// them). Conservative `false` for anything else. -/// -/// Rooted is not address-STABLE, though: the copying minor moves -/// shape-shared arrays like anything else (`move_young` merely preserves -/// the flag), rewriting every rooted reference — but not the `@perry_ic_N` -/// globals, which no GC scanner knows about. The vacated from-space address -/// is then recycled, and a different keys array landing there makes a -/// primed site falsely HIT with the old slot mapping (#6080a). That residual -/// is closed by [`PERRY_IC_EPOCH`] below, not by this predicate. -pub(crate) unsafe fn keys_cacheable_for_pic(keys: *const crate::array::ArrayHeader) -> bool { - let Some(gc) = crate::value::addr_class::try_read_gc_header(keys as usize) else { - return false; - }; - gc.obj_type == crate::gc::GC_TYPE_ARRAY && gc.gc_flags & crate::gc::GC_FLAG_SHAPE_SHARED != 0 -} - -/// #6080(a): process-global read-PIC epoch, exported to the emitted IR as -/// `@PERRY_IC_EPOCH` (same pattern as `PERRY_TA_VIEW_GUARD`). A keys-POINTER -/// token primed into a `@perry_ic_N` cache is only trustworthy for as long -/// as no address has been freed or moved since priming: the cache global is -/// invisible to every GC scanner, so a recycled keys-array address would -/// pointer-match a different shape and the inline hit path would load the -/// wrong slot — silently. -/// -/// The miss handler snapshots this epoch into `cache[2]` at prime time; the -/// emitted hit predicate requires `cache[2] == PERRY_IC_EPOCH` before -/// trusting a pointer token (shape-ID tokens skip the check — ids are never -/// reused, so they cannot alias). Every completed collection bumps the epoch -/// (`GcStats::record_collection`, the single per-collection funnel), and -/// budgeted cycles additionally bump at sweep ENTRY, because their sweep -/// slices interleave with the mutator — an address freed by an early slice -/// must not be trusted while the cycle is still running. -/// -/// Starts at 1 so a `zeroinitializer` cache (epoch 0) can never match. -#[no_mangle] -pub static PERRY_IC_EPOCH: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); - -/// Invalidate every pointer-token read-PIC prime (see [`PERRY_IC_EPOCH`]). -pub(crate) fn pic_epoch_bump() { - PERRY_IC_EPOCH.fetch_add(1, std::sync::atomic::Ordering::Relaxed); -} - /// Words in a per-site property-read cache global (`@perry_ic_N`). Codegen /// emits `[PIC_CACHE_WORDS x i64] zeroinitializer`; this type is the runtime's /// view of the same memory. @@ -243,9 +197,9 @@ pub const PIC_CACHE_WORDS: usize = 12; /// /// | word | meaning | /// |---|---| -/// | 0 | `tok0` — most-recently-used shape token (ID token **or** keys pointer) | +/// | 0 | `tok0` — most-recently-used ShapeId token | /// | 1 | `slot0` — its resolved field slot | -/// | 2 | `epoch` — [`PERRY_IC_EPOCH`] snapshot; gates word 0's pointer tokens **and every way** | +/// | 2 | reserved non-identity scratch | /// | 3,4 / 5,6 / 7,8 / 9,10 | `(tok, slot)` ways | /// | 11 | round-robin victim index for the ways | pub type PicCache = [i64; PIC_CACHE_WORDS]; @@ -266,7 +220,7 @@ pub(crate) const PIC_WAYS: usize = 4; /// /// | value | meaning | emitted code | /// |---|---|---| -/// | `0` | no way is populated (fresh site, or just epoch-wiped) | skip the compares | +/// | `0` | no way is populated (fresh site) | skip the compares | /// | `> 0` | armed: bit 0 set, bits 1..7 the round-robin victim, bits 8.. the *consecutive* capacity-eviction run | run the compares | /// | `< 0` | **megamorphic** — the rotation is wider than the ways hold. The magnitude is a countdown: each further miss adds 1, and at 0 the site is armed again | skip the compares | pub(crate) const PIC_WAY_STATE: usize = 3; @@ -308,50 +262,19 @@ const PIC_LATCH_RETRY: i64 = 2048; /// resolves it inline instead of calling back into this handler. A site that /// alternates between k ≤ `PIC_WAYS + 1` shapes therefore stops thrashing. /// -/// Both token kinds are cascaded, because the population that matters is the -/// pointer-token one: a plain object literal is allocated through a generated -/// `__AnonShape_*` constructor and so carries a real `class_id`, which routes it -/// to the shape-shared keys-POINTER prime, not the `#6804` shape-ID prime. Ways -/// restricted to ID tokens are dead code for exactly the programs this exists -/// for — measured as a 6% *regression* on a tree-walking interpreter, all of it -/// the compare sequence running and never hitting. -/// -/// A keys-POINTER token is address-derived and can be recycled after a -/// collection (#6080a), so every way is gated on the SAME `cache[2]` epoch word -/// the MRU entry uses, and this function **wipes the ways whenever the epoch -/// moves**. That keeps the shared word honest: a way is only ever readable while -/// `cache[2]` still holds the epoch that way was primed in. The ways go cold -/// once per collection and re-prime — 38 minor collections across a 4 s run, so -/// the re-priming is not measurable. -/// -/// `(shape, key)` → slot is immutable within an epoch: a site always looks up -/// one key, and a keys-array change gives the object a different keys array (or -/// a fresh shape id). So a way that stops matching simply goes cold; it can -/// never resolve to a wrong slot. +/// Every token is derived from an authoritative, never-reused ShapeId. A shape +/// transition therefore makes an old way go cold without requiring an address +/// epoch or any GC-visible cache rewriting. /// /// # Safety /// `cache` must point at a live `[i64; PIC_CACHE_WORDS]` (the codegen-emitted /// per-site global, or a stack array of that type). -pub(crate) unsafe fn pic_prime_get(cache: *mut PicCache, token: i64, slot: i64, epoch: i64) { +pub(crate) unsafe fn pic_prime_get(cache: *mut PicCache, token: i64, slot: i64) { let c = &mut *cache; let prev_tok = c[0]; let prev_slot = c[1]; - // A collection happened since this site was last primed. Every token here — - // word 0's and every way's — was resolved against addresses that may since - // have been freed, moved and recycled, so the whole cache goes cold. That - // includes `prev_tok`: cascading it would smuggle a stale pointer token past - // the very guard the wipe exists to enforce. - let epoch_held = c[2] == epoch; c[0] = token; c[1] = slot; - c[2] = epoch; - if !epoch_held && c[PIC_WAY_STATE] >= 0 { - for w in 0..PIC_WAYS { - c[PIC_WAY_BASE + w * 2] = 0; - c[PIC_WAY_BASE + w * 2 + 1] = 0; - } - c[PIC_WAY_STATE] = 0; - } // Megamorphic. A rotation wider than the ways hold never hits one, so the // compare sequence becomes pure cost — measured at **+37%** on a 7-shape // site, against a 2.5x SPEEDUP on a 5-shape one. That asymmetry is the whole @@ -368,7 +291,7 @@ pub(crate) unsafe fn pic_prime_get(cache: *mut PicCache, token: i64, slot: i64, c[PIC_WAY_STATE] = state + 1; return; } - let cascade = epoch_held && prev_tok != 0 && prev_tok != token; + let cascade = prev_tok != 0 && prev_tok != token; // One pass over the ways does three things: // * evicts `token` from a way if it has one — it now lives in the MRU // entry, and leaving the stale copy behind would permanently cost a way @@ -458,20 +381,18 @@ unsafe fn key_bytes_are(key: *const crate::StringHeader, want: &[u8]) -> bool { /// Monomorphic inline cache miss handler (issue #51). /// -/// Called when the codegen-emitted shape check (`obj->keys_array == cache[0]`) +/// Called when the codegen-emitted ShapeId check misses. /// fails. Performs the full field lookup via `js_object_get_field_by_name`, /// then populates the per-site cache so subsequent calls with the same shape /// hit the inline fast path (no function call, direct field load). /// -/// `cache` layout: see [`PicCache`]. Words 0..2 are the MRU entry -/// `[shape_token, field_slot_index, primed_epoch]` (`shape_token` is a shape-ID -/// token or a raw keys-array pointer — see #6804; `primed_epoch` is the -/// [`PERRY_IC_EPOCH`] snapshot taken at prime time, #6080a); words 3.. are the -/// polymorphic ways filled by [`pic_prime_get`] (#7753). +/// `cache` layout: see [`PicCache`]. Words 0..1 are the ShapeId-token MRU entry; +/// word 2 is reserved scratch, and words 3.. are the polymorphic ways filled by +/// [`pic_prime_get`] (#7753). /// /// Only caches when: /// - obj is a valid ObjectHeader (not null, not handle, not string/array/etc.) -/// - field exists and its slot index < 8 (inline allocation limit) +/// - field exists and its slot index is below `shape.live_inline_slot_count` /// /// Overflow fields (slot >= alloc_limit) are NOT cached and fall through to /// the slow path — the fast path loads from `obj_ptr + 24 + slot*8` which @@ -691,32 +612,24 @@ pub extern "C" fn js_object_get_field_ic_miss( (*gc_header).obj_type == crate::gc::GC_TYPE_OBJECT }; let has_own_descriptors = is_object && super::super::object_has_descriptors(obj as usize); - let is_regular = is_object && (*obj).object_type == crate::error::OBJECT_TYPE_REGULAR; + let is_regular = is_object && crate::object::object_is_regular(obj); // Gate-neutral builtin accessors deliberately leave the process-wide // accessor latch clear. Their owner bit must still block this PIC: // its generated hit path is a raw slot load and would otherwise turn // `Set.prototype.size` into `undefined` instead of invoking the getter. if can_cache && is_regular && !has_own_descriptors { - let keys = (*obj).keys_array; + let Some(shape) = crate::object::shapes::object_shape_descriptor(obj) else { + let value = js_object_get_field_by_name(obj, key); + return f64::from_bits(value.bits()); + }; + let keys = shape.keys as usize as *mut crate::array::ArrayHeader; if keys.is_null() || (keys as usize) <= 0x10000 { let value = js_object_get_field_by_name(obj, key); return f64::from_bits(value.bits()); } - let key_count = *(keys as *const u32) as usize; + let key_count = shape.logical_key_count as usize; let keys_data = (keys as *const u8).add(8) as *const f64; - let alloc_limit = - std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32) as usize; - // #6804: stamp the receiver's stable ShapeId at PIC-miss - // resolution, so the id-keyed FIELD_CACHE (and the future - // id-comparing PIC) see a stamped object from its first read. - // #6759 C3 rung 1: class instances are stamped here too. - if crate::object::shapes::object_shape_stamp(obj) == 0 { - crate::object::shapes::stamp_object_shape( - obj as *mut ObjectHeader, - keys, - key_count as u32, - ); - } + let alloc_limit = shape.live_inline_slot_count as usize; for i in 0..key_count { let k_bits = (*keys_data.add(i)).to_bits(); let k_ptr = (k_bits & 0x0000_FFFF_FFFF_FFFF) as *const crate::StringHeader; @@ -740,38 +653,11 @@ pub extern "C" fn js_object_get_field_ic_miss( // ~900k times per run (40% inclusive samples per // perfcomp.profile). // - // #6804: a stamped plain receiver primes an ID token - // (`stamp | PIC_ID_TOKEN_BIT`, matching the emitted - // PIC's discriminated compare). Ids are never reused, - // so id tokens are immune to the address-recycling ABA - // that keys-pointer tokens have — which also makes - // OWNED keys arrays safely cacheable again for plain - // objects. #6759 C3c: keys-POINTER tokens stay - // restricted to SHAPE-SHARED arrays (literal shapes, - // class-keys arrays — shape-cache-resident, - // process-rooted, address-stable), because that compare - // is unvalidated and a recycled owned-array address - // would read the wrong slot. - // - // #6759 C3 rung 1: `object_shape_stamp` carries no - // `class_id` discriminant, so a stamped CLASS INSTANCE - // primes an id token too — which is what the emitted PIC - // already computes for it (its `is_stamp` test is the - // range test alone). Priming the keys pointer for a - // stamped receiver would be a permanent miss. - let stamp = crate::object::shapes::object_shape_stamp(obj); - // #6080a: stamp the current GC epoch alongside either - // token kind. The emitted hit predicate only consults it - // for pointer tokens, but priming it unconditionally - // keeps `cache[2]` coherent when a site re-primes from - // one token kind to the other. - let epoch = PERRY_IC_EPOCH.load(std::sync::atomic::Ordering::Relaxed) as i64; - if stamp != 0 { - let token = (stamp as u64 | crate::object::shapes::PIC_ID_TOKEN_BIT) as i64; - pic_prime_get(cache, token, i as i64, epoch); - } else if keys_cacheable_for_pic(keys) { - pic_prime_get(cache, keys as i64, i as i64, epoch); - } + // The runtime and emitted hit path share one identity: + // the authoritative, never-reused ShapeId token. + let stamp = crate::object::shapes::object_shape_id(obj); + let token = (stamp as u64 | crate::object::shapes::PIC_ID_TOKEN_BIT) as i64; + pic_prime_get(cache, token, i as i64); let field_ptr = (obj as *const u8) .add(std::mem::size_of::() + i * 8) as *const f64; @@ -1113,20 +999,19 @@ mod poly_pic_tests { ); } - /// The MRU entry keeps its pre-#7753 meaning exactly: always overwritten, - /// carrying its epoch. A monomorphic site must therefore look identical to - /// what it looked like before the ways existed — no way is ever filled. + /// The MRU entry is always overwritten. A monomorphic site never fills a + /// polymorphic way, and its reserved scratch word stays non-identifying. #[test] fn monomorphic_site_never_fills_a_way() { let mut c: PicCache = [0; PIC_CACHE_WORDS]; unsafe { for _ in 0..8 { - pic_prime_get(&mut c, id_tok(7), 2, 99); + pic_prime_get(&mut c, id_tok(7), 2); } } assert_eq!(c[0], id_tok(7)); assert_eq!(c[1], 2); - assert_eq!(c[2], 99); + assert_eq!(c[2], 0); for w in 0..PIC_WAYS { assert_eq!( c[PIC_WAY_BASE + w * 2], @@ -1151,7 +1036,7 @@ mod poly_pic_tests { // Two full rotations: the first fills, the second must not disturb. for _ in 0..2 { for (tok, slot) in &shapes { - pic_prime_get(&mut c, *tok, *slot, 1); + pic_prime_get(&mut c, *tok, *slot); } } } @@ -1198,7 +1083,7 @@ mod poly_pic_tests { unsafe { for _ in 0..40 { for (slot, tok) in shapes.iter().enumerate() { - pic_prime_get(&mut c, *tok, slot as i64, 3); + pic_prime_get(&mut c, *tok, slot as i64); } } } @@ -1217,7 +1102,7 @@ mod poly_pic_tests { let latched = c[PIC_WAY_STATE]; unsafe { for _ in 0..8 { - pic_prime_get(&mut c, shapes[0], 0, 3); + pic_prime_get(&mut c, shapes[0], 0); } } assert!(c[PIC_WAY_STATE] < 0, "the latch must not clear immediately"); @@ -1236,11 +1121,11 @@ mod poly_pic_tests { // a phase change cannot kill it for the rest of the process. unsafe { while c[PIC_WAY_STATE] < 0 { - pic_prime_get(&mut c, shapes[0], 0, 3); + pic_prime_get(&mut c, shapes[0], 0); } // Two shapes is well inside capacity: the ways must fill again. - pic_prime_get(&mut c, shapes[1], 1, 3); - pic_prime_get(&mut c, shapes[0], 0, 3); + pic_prime_get(&mut c, shapes[1], 1); + pic_prime_get(&mut c, shapes[0], 0); } assert!( c[PIC_WAY_STATE] > 0, @@ -1260,7 +1145,7 @@ mod poly_pic_tests { unsafe { for _ in 0..200 { for i in 0..(PIC_WAYS as i64 + 1) { - pic_prime_get(&mut c, 0x6000_0000_0000 + i * 8, i, 4); + pic_prime_get(&mut c, 0x6000_0000_0000 + i * 8, i); } } } @@ -1289,11 +1174,11 @@ mod poly_pic_tests { unsafe { for round in 0..400 { for (slot, tok) in hot.iter().enumerate() { - pic_prime_get(&mut c, *tok, slot as i64, 5); + pic_prime_get(&mut c, *tok, slot as i64); } // One interloper every round — far more than the 80 the // interpreter produced, and 10x the raw threshold. - pic_prime_get(&mut c, 0x7000_FFFF_0000 + round, 0, 5); + pic_prime_get(&mut c, 0x7000_FFFF_0000 + round, 0); } } assert!( @@ -1302,68 +1187,25 @@ mod poly_pic_tests { ); } - /// The population that matters is the keys-POINTER one: a plain object - /// literal goes through a generated `__AnonShape_*` constructor, so it has - /// a real `class_id` and primes a keys pointer, never a shape id. Ways that - /// only accept ID tokens are dead code for exactly the programs the ways - /// exist for (measured: a 6% regression, the compares running and never - /// hitting). This is the test that would have caught shipping that. + /// ShapeId tokens cascade into the polymorphic ways without losing their + /// paired slot. #[test] - fn pointer_tokens_do_reach_a_way() { + fn shape_id_tokens_do_reach_a_way() { let mut c: PicCache = [0; PIC_CACHE_WORDS]; - let ptr_a = 0x2000_1234_5678_i64; - let ptr_b = 0x2000_1234_9999_i64; - assert_eq!((ptr_a as u64) & PIC_ID_TOKEN_BIT, 0, "test premise"); + let shape_a = id_tok(41); + let shape_b = id_tok(42); unsafe { - pic_prime_get(&mut c, ptr_a, 1, 5); - pic_prime_get(&mut c, ptr_b, 2, 5); + pic_prime_get(&mut c, shape_a, 1); + pic_prime_get(&mut c, shape_b, 2); } - assert_eq!(c[0], ptr_b); + assert_eq!(c[0], shape_b); assert!( (0..PIC_WAYS) - .any(|w| c[PIC_WAY_BASE + w * 2] == ptr_a && c[PIC_WAY_BASE + w * 2 + 1] == 1), - "the evicted keys-pointer token must land in a way: {c:?}" + .any(|w| c[PIC_WAY_BASE + w * 2] == shape_a && c[PIC_WAY_BASE + w * 2 + 1] == 1), + "the evicted ShapeId token must land in a way: {c:?}" ); } - /// #6080a, extended to the ways. A keys-POINTER token is an ADDRESS: after a - /// collection frees or moves that keys array, a different-shape array can be - /// recycled into the same address and a stale way would pointer-match and - /// load the wrong slot — silently, which is the worst failure this code can - /// have. The ways share word 2's epoch snapshot with the MRU entry, so the - /// discipline that makes that sound is: a new epoch WIPES every way, and the - /// token being evicted from word 0 is dropped rather than cascaded (it too - /// was resolved in the old epoch). This asserts both halves. - #[test] - fn an_epoch_change_wipes_every_way() { - let mut c: PicCache = [0; PIC_CACHE_WORDS]; - unsafe { - for i in 0..(PIC_WAYS as i64 + 1) { - pic_prime_get(&mut c, 0x3000_0000_0000 + i, i, 7); - } - } - assert!( - (0..PIC_WAYS).any(|w| c[PIC_WAY_BASE + w * 2] != 0), - "test premise: the ways are populated before the epoch moves" - ); - let stale = c[0]; - unsafe { - pic_prime_get(&mut c, 0x4000_0000_0000, 3, 8); - } - for w in 0..PIC_WAYS { - assert_eq!( - c[PIC_WAY_BASE + w * 2], - 0, - "way {w} survived an epoch change: {c:?}" - ); - } - assert_ne!( - c[0], stale, - "the MRU entry must hold the freshly primed token" - ); - assert_eq!(c[2], 8, "word 2 must carry the new epoch"); - } - /// More distinct shapes than the site can hold must degrade to "some miss", /// never to a wrong answer: every occupied way still carries the slot it was /// primed with, so the emitted compare can only hit on a token it stored. @@ -1372,7 +1214,7 @@ mod poly_pic_tests { let mut c: PicCache = [0; PIC_CACHE_WORDS]; unsafe { for i in 0..(PIC_WAYS as u64 * 4) { - pic_prime_get(&mut c, id_tok(200 + i), i as i64, 1); + pic_prime_get(&mut c, id_tok(200 + i), i as i64); } } for w in 0..PIC_WAYS { @@ -1392,81 +1234,12 @@ mod poly_pic_tests { #[cfg(test)] mod c3c_pic_tests { - /// #6759 C3c: the PIC only caches SHAPE-SHARED (process-rooted, - /// address-stable) keys arrays; an owned array's address can be - /// recycled under a different shape, which the unvalidated PIC hit - /// path cannot detect. - #[test] - fn pic_caches_only_shape_shared_keys() { - let _lock = crate::gc::global_side_table_test_lock(); - unsafe { - let keys = crate::array::js_array_alloc(4); - assert!( - !super::keys_cacheable_for_pic(keys), - "a fresh owned keys array must not be PIC-cacheable" - ); - let gc = (keys as *const u8).sub(crate::gc::GC_HEADER_SIZE) as *mut crate::gc::GcHeader; - (*gc).gc_flags |= crate::gc::GC_FLAG_SHAPE_SHARED; - assert!( - super::keys_cacheable_for_pic(keys), - "a shape-shared keys array must stay PIC-cacheable" - ); - } - } - - /// #6080a: a pointer-token prime snapshots the live PIC epoch into - /// `cache[2]`, and a subsequent epoch bump strands that snapshot — the - /// exact inputs of the emitted `cache[2] == @PERRY_IC_EPOCH` guard, so - /// this proves the guard CAN fail (a primed entry goes stale), not just - /// that priming writes something. - /// - /// ★ #6759 C3 rung 1 shrank this path's PRODUCTION population to nothing - /// reachable from source. Class instances used to be the last receivers - /// priming a raw keys pointer (plain objects took the #6804 id token since - /// then); rung 1 stamps them too, so `js_object_get_field_ic_miss` now - /// mints-then-primes an id for every receiver whose mint succeeds. The - /// pointer arm survives as the id-exhaustion fallback (`alloc_shape_id` - /// returns 0 after 2^30 shape births) and as what the emitted hit - /// predicate still computes for an as-yet-unstamped receiver — neither is - /// constructible from a `.ts` fixture, so the epoch mechanics are driven - /// through `pic_prime_get` directly. The end-to-end half below asserts the - /// rung-1 behaviour instead: a class instance primes an ID token, which is - /// what the emitted PIC computes for it once it carries a stamp. - #[test] - fn pointer_token_prime_stamps_epoch_and_goes_stale_on_bump() { - let _lock = crate::gc::global_side_table_test_lock(); - unsafe { - use std::sync::atomic::Ordering; - // The pointer arm, driven directly (see the note above). - let mut cache = [0i64; super::PIC_CACHE_WORDS]; - let fake_keys = 0x6080_0000_1000i64; - let live = super::PERRY_IC_EPOCH.load(Ordering::Relaxed) as i64; - super::pic_prime_get(&mut cache, fake_keys, 3, live); - assert_eq!(cache[0], fake_keys, "pointer token must land in way 0"); - assert_eq!(cache[2], live, "prime must snapshot the LIVE epoch"); - assert!(cache[2] >= 1, "epoch starts at 1, never 0"); - - super::pic_epoch_bump(); - assert_ne!( - cache[2], - super::PERRY_IC_EPOCH.load(Ordering::Relaxed) as i64, - "a bump must strand every pointer-token prime (the emitted \ - hit predicate then misses and re-primes)" - ); - } - } - - /// #6759 C3 rung 1: a CLASS instance is stamped at its first by-name - /// resolve and therefore primes an ID token, not its keys pointer. The - /// emitted PIC discriminates on the ShapeId RANGE alone (it never loads - /// `class_id` for this), so priming the keys pointer for a stamped - /// receiver would be a permanent miss — this test is what keeps the - /// runtime's choice and the IR's choice the same. + /// A class instance primes the same authoritative ShapeId token that the + /// emitted guard reads from the receiver. #[test] fn a_class_instance_primes_an_id_token_after_rung1() { let _lock = crate::gc::global_side_table_test_lock(); unsafe { - use std::sync::atomic::Ordering; let obj = crate::object::js_object_alloc(0x6080, 8); let key = crate::string::js_string_from_bytes(b"pic6080_x".as_ptr(), 9); crate::object::js_object_set_field_by_name(obj, key, 7.0); @@ -1494,21 +1267,15 @@ mod c3c_pic_tests { "primed the keys pointer for a stamped receiver — every hit at \ this site would miss forever" ); - assert_eq!( - cache[2], - super::PERRY_IC_EPOCH.load(Ordering::Relaxed) as i64, - "cache[2] must stay coherent across token kinds" - ); + assert_eq!(cache[2], 0, "word 2 is non-identity scratch"); } } /// ★ #6759 C3 rung 1 opens a NEW correctness surface, and this is it. /// - /// Before rung 1 a delete-compacted class instance was UNCACHEABLE by the - /// read PIC: its keys array is a private clone, so `keys_cacheable_for_pic` - /// (SHAPE_SHARED only) refused it and the site fell through to the slow - /// path forever. Rung 1 stamps it, so it primes an id token and the emitted - /// hit path starts serving it. A token that failed to move across the + /// A delete-compacted class instance receives a semantic successor ShapeId, + /// so the emitted hit path can serve it without confusing it with a + /// pristine sibling. A token that failed to move across the /// compaction would therefore be read as a pristine sibling's shape at a /// site that has both — the one-slot shift the whole ladder is about. /// @@ -1580,20 +1347,15 @@ mod c3c_pic_tests { /// from `perry-codegen/src/expr/property_get/generic_dispatch.rs`: /// /// ```text - /// is_stamp = (parent_class_id - 0x8000_0000) u< 0x4000_0000 - /// token = is_stamp ? (parent_class_id | 1<<62) : keys_array + /// token = valid_shape_id ? (shape_id | 1<<62) : 0 /// ``` /// /// The runtime never calls this; it exists so a test can compare what the /// miss handler PRIMES against what the hit path will COMPUTE, which is /// the only pair whose agreement decides whether a site can ever hit. unsafe fn emitted_pic_token(obj: *const super::ObjectHeader) -> u64 { - let word = (*obj).parent_class_id; - if crate::object::shapes::is_shape_id(word) { - word as u64 | crate::object::shapes::PIC_ID_TOKEN_BIT - } else { - (*obj).keys_array as u64 - } + let shape_id = crate::object::shapes::object_shape_id(obj); + shape_id as u64 | crate::object::shapes::PIC_ID_TOKEN_BIT } /// ★ The invariant #6759 C3 rung 1 broke, asserted where it broke. @@ -1601,12 +1363,10 @@ mod c3c_pic_tests { /// A shape's population must be UNIFORMLY stamped: the token the miss /// handler primes from one instance is only useful if a DIFFERENT, /// freshly-allocated instance of the same class computes the same token. - /// Rung 1 (#7983) stamped class instances lazily while their allocator - /// still wrote the real `parent_class_id`, so instance #1 primed an id - /// token and every newborn sibling computed its keys pointer instead — - /// `token_eq` failed at every site reading a field of a fresh instance, - /// forever. Measured cost before the birth stamp: `cycles` +54%, - /// `deeplist` +45%, `interp` +28% in instructions retired. + /// A prior implementation stamped class instances lazily, so instance #1 + /// primed an id token while every newborn sibling computed a different + /// identity. `token_eq` then failed at every field-read site until the + /// sibling took the miss path itself. /// /// This is deliberately NOT "the newborn carries a stamp" — that is a /// presence check two different states satisfy (both-stamped and diff --git a/crates/perry-runtime/src/object/field_set_by_name.rs b/crates/perry-runtime/src/object/field_set_by_name.rs index 5d7c263187..bae615a342 100644 --- a/crates/perry-runtime/src/object/field_set_by_name.rs +++ b/crates/perry-runtime/src/object/field_set_by_name.rs @@ -132,7 +132,7 @@ pub extern "C" fn js_object_set_field_by_name( // completion runs, and (template cid, key) plans recorded // by instances sharing the cid would falsely certify it. // See the matching gates at the plan record sites. - if (*o).object_type == crate::error::OBJECT_TYPE_REGULAR + if crate::object::object_is_regular(o) && class_id != 0 && class_id != NATIVE_MODULE_CLASS_ID && !super::prototype_chain::object_has_prototype_override(raw) diff --git a/crates/perry-runtime/src/object/field_set_by_name/fast_paths.rs b/crates/perry-runtime/src/object/field_set_by_name/fast_paths.rs index 15b69d8763..53162e8adf 100644 --- a/crates/perry-runtime/src/object/field_set_by_name/fast_paths.rs +++ b/crates/perry-runtime/src/object/field_set_by_name/fast_paths.rs @@ -38,7 +38,7 @@ pub(crate) unsafe fn try_existing_own_data_overwrite( if obj_gc.obj_type != crate::gc::GC_TYPE_OBJECT || obj_gc.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 || obj_gc._reserved & BLOCKING_FLAGS != 0 - || (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR + || !crate::object::object_is_regular(obj) || (*obj).class_id == NATIVE_MODULE_CLASS_ID || crate::array::object_prototype_addr_matches(obj_addr) // URL's visible fields are live views over one backing URL. An own @@ -196,9 +196,7 @@ pub extern "C" fn js_object_set_field_by_name_transition_fast( { return 0; } - if (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR - || (*obj).class_id == NATIVE_MODULE_CLASS_ID - { + if !crate::object::object_is_regular(obj) || (*obj).class_id == NATIVE_MODULE_CLASS_ID { return 0; } diff --git a/crates/perry-runtime/src/object/field_set_by_name/tail.rs b/crates/perry-runtime/src/object/field_set_by_name/tail.rs index 3a09f15a69..e0bec0aa48 100644 --- a/crates/perry-runtime/src/object/field_set_by_name/tail.rs +++ b/crates/perry-runtime/src/object/field_set_by_name/tail.rs @@ -49,9 +49,9 @@ pub(super) fn set_field_by_name_object_tail( let mut value = value_handle.get_nanbox_f64(); // Safety: obj is a valid heap pointer (> 0x10000) at this point unsafe { - // Validate this is an ObjectHeader, not some other heap type. - // Check GcHeader first (reliable for heap objects), then fallback to ObjectHeader.object_type - // for static/const objects that don't have GcHeaders. + // Validate this is an ObjectHeader, not some other heap type. Every + // shaped object has a tracked GcHeader; payload `object_type` is only + // a compatibility mirror and is never a kind fallback. // Guard: ensure we can safely read GC_HEADER_SIZE bytes before obj if (obj as usize) < crate::gc::GC_HEADER_SIZE + 0x1000 { return; @@ -160,18 +160,7 @@ pub(super) fn set_field_by_name_object_tail( // had MapHeader.size aliasing object_type == OBJECT_TYPE_REGULAR, // so `m.customProp = 5` walked the Map's bytes as object fields — // deterministic heap corruption (2026-07-02 audit P1). The - // object_type fallback exists ONLY for static/const objects whose - // preceding bytes decode to no known GC type. - if crate::gc::gc_type_info(gc_type).is_some() { - return; - } - if !is_valid_obj_ptr(obj as *const u8) { - return; - } - let object_type = (*obj).object_type; - if object_type != crate::error::OBJECT_TYPE_REGULAR { - return; - } + return; } if gc_type == crate::gc::GC_TYPE_CLOSURE { @@ -263,14 +252,15 @@ pub(super) fn set_field_by_name_object_tail( const PLAN_BLOCKING_FLAGS: u16 = crate::gc::OBJ_FLAG_NULL_PROTO | crate::gc::OBJ_FLAG_HAS_DESCRIPTORS; let obj_class_id = (*obj).class_id; - // #6595: class objects (`OBJECT_TYPE_CLASS`) are excluded — their + // #6595: class objects are excluded by their authoritative ShapeId + // kind — their // writes must always reach the `mirror_class_object_static_write` // completions, and their cid is shared with their instances so a // plan keyed on it conflates two different prototype chains. let plan_eligible = !key.is_null() && obj_class_id != 0 && obj_class_id != NATIVE_MODULE_CLASS_ID - && (*obj).object_type == crate::error::OBJECT_TYPE_REGULAR + && crate::object::object_is_regular(obj) && (*gc_header)._reserved & PLAN_BLOCKING_FLAGS == 0 && !super::prototype_chain::object_has_prototype_override(obj as usize); let plan_fast = plan_eligible @@ -441,12 +431,12 @@ pub(super) fn set_field_by_name_object_tail( // cleared the chain. Record the verdict so the next store skips // the vet (`plan_fast` above). Eligibility is re-derived from the // freshly read `obj_flags`, not the pre-vet read. - if !plan_fast - && obj_class_id != 0 + let record_plan_eligible = obj_class_id != 0 && obj_class_id != NATIVE_MODULE_CLASS_ID - && (*obj).object_type == crate::error::OBJECT_TYPE_REGULAR + && crate::object::object_is_regular(obj) && obj_flags & PLAN_BLOCKING_FLAGS == 0 - { + && !super::prototype_chain::object_has_prototype_override(obj as usize); + if !plan_fast && record_plan_eligible { super::prop_plan::store_plan_record(obj_class_id, interned_key as usize); } if let Some((next_keys, slot_idx)) = diff --git a/crates/perry-runtime/src/object/field_set_by_name/write_helpers.rs b/crates/perry-runtime/src/object/field_set_by_name/write_helpers.rs index ceaa4ddffe..db63cdffee 100644 --- a/crates/perry-runtime/src/object/field_set_by_name/write_helpers.rs +++ b/crates/perry-runtime/src/object/field_set_by_name/write_helpers.rs @@ -68,7 +68,7 @@ pub(super) unsafe fn mirror_class_object_static_write( key: *const crate::StringHeader, value: f64, ) { - if (*obj).object_type != crate::error::OBJECT_TYPE_CLASS + if !crate::object::is_class_object_ptr(obj as *const u8) || (*obj).class_id == 0 || key.is_null() { diff --git a/crates/perry-runtime/src/object/gc_slots.rs b/crates/perry-runtime/src/object/gc_slots.rs new file mode 100644 index 0000000000..d3f1971971 --- /dev/null +++ b/crates/perry-runtime/src/object/gc_slots.rs @@ -0,0 +1,72 @@ +use super::{shapes, ObjectHeader}; +use crate::ArrayHeader; + +pub(crate) unsafe fn gc_keys_array_slot(obj: *mut ObjectHeader) -> Option<*mut u64> { + if obj.is_null() { + return None; + } + if let Some(descriptor) = shapes::object_shape_descriptor(obj) { + // Compatibility scratch slot: GC obtains the authoritative edge from + // the ShapeId descriptor, then lets the existing slot visitor rewrite + // it in place. #8047 can replace this scratch with a descriptor-table + // rewrite without changing the source of the edge. + // + // The descriptor lookup immediately precedes this collector-side + // materialization: no allocation or callback can change its `keys` + // edge before the visitor receives the slot. That ordering is what + // makes the descriptor authoritative while this legacy field remains + // only the mutable scratch location used for pointer rewriting. + // GC_STORE_AUDIT(ROOT): collector materializes the authoritative descriptor root into its compatibility rewrite slot. + (*obj).keys_array = descriptor.keys as usize as *mut ArrayHeader; + } + if (*obj).keys_array.is_null() { + return None; + } + Some(&mut (*obj).keys_array as *mut _ as *mut u64) +} + +pub(crate) unsafe fn gc_field_slot_range( + obj: *mut ObjectHeader, +) -> Option { + if obj.is_null() { + return None; + } + let field_count = shapes::object_shape_descriptor(obj) + .map(|descriptor| descriptor.live_inline_slot_count as usize) + // Compatibility only for synthetic/raw test fixtures that bypass all + // runtime allocators. Published runtime objects are always stamped. + .unwrap_or((*obj).field_count as usize); + if field_count > 1_000_000 { + return None; + } + let fields = (obj as *mut u8).add(std::mem::size_of::()) as *mut u64; + Some(crate::gc::HeapSlotRange::new(fields, field_count)) +} + +#[inline] +pub(crate) unsafe fn rebuild_object_field_layout(obj: *mut ObjectHeader, slot_count: usize) { + let fields = (obj as *mut u8).add(std::mem::size_of::()) as *mut u64; + crate::gc::layout_rebuild_from_slots(obj as *mut u8, fields, slot_count); + if crate::arena::pointer_in_old_gen(obj as usize) { + for i in 0..slot_count { + let slot = fields.add(i); + crate::gc::runtime_write_barrier_slot(obj as usize, slot as usize, *slot); + } + } +} + +#[inline] +pub(crate) unsafe fn rebuild_array_layout_from_slots(arr: *mut ArrayHeader) { + if arr.is_null() { + return; + } + let len = (*arr).length as usize; + let slots = (arr as *mut u8).add(std::mem::size_of::()) as *mut u64; + crate::gc::layout_rebuild_from_slots(arr as *mut u8, slots, len); + if crate::arena::pointer_in_old_gen(arr as usize) { + for i in 0..len { + let slot = slots.add(i); + crate::gc::runtime_write_barrier_slot(arr as usize, slot as usize, *slot); + } + } +} diff --git a/crates/perry-runtime/src/object/mod.rs b/crates/perry-runtime/src/object/mod.rs index 0b14d04640..2041ac1405 100644 --- a/crates/perry-runtime/src/object/mod.rs +++ b/crates/perry-runtime/src/object/mod.rs @@ -73,9 +73,13 @@ mod descriptors; mod disposable_proto_thunks; pub(crate) mod exotic_expando; mod field_get_set; -pub(crate) use field_get_set::pic_epoch_bump; pub(crate) use field_get_set::scan_accessor_receiver_override_root_mut; mod field_set_by_name; +mod gc_slots; +pub(crate) use gc_slots::{ + gc_field_slot_range, gc_keys_array_slot, rebuild_array_layout_from_slots, + rebuild_object_field_layout, +}; mod global_fetch; pub(crate) use global_fetch::scan_pending_fetch_signal_root_mut; mod global_this; @@ -1709,11 +1713,9 @@ pub struct ObjectHeader { /// free paths, no owner registry, and no stale-address hazard: the record /// dies with (and only with) its owner. /// -/// CAUTION — RegExp aliasing: `RegExpHeader` is a different struct that is -/// also tagged `GC_TYPE_OBJECT` (see `gc_child_slots`'s regex special -/// case). Reading `.meta` at the `ObjectHeader` offset off a RegExp yields -/// garbage; every `meta` access must first establish a genuine shaped -/// object (`object_meta_slot_addr` centralizes that check). +/// Only the authoritative `GC_TYPE_OBJECT` kind has this layout. RegExp uses +/// its own GC kind and slot descriptor, so no ObjectHeader consumer needs to +/// inspect its native payload to disambiguate the two. /// /// The shipped Phase B record holds the custom `[[Prototype]]`, the Phase C2 /// per-key descriptor summaries, object flags, and owned spill storage. The @@ -1763,6 +1765,36 @@ pub struct ObjectMeta { pub(crate) const OBJECT_META_FLAG_PROTO_OVERRIDE: u64 = 1; +/// Authoritative ordinary-object discriminator. RegExp has its own GC kind, +/// and heap class-expression values carry their kind in the immutable ShapeId +/// descriptor. The legacy `ObjectHeader::object_type` word is only an ABI +/// mirror pending #8047. +#[inline] +pub(crate) unsafe fn object_is_regular(obj: *const ObjectHeader) -> bool { + if obj.is_null() { + return false; + } + let Some(header) = crate::value::addr_class::try_read_gc_header(obj as usize) else { + return false; + }; + header.obj_type == crate::gc::GC_TYPE_OBJECT + && header.gc_flags & crate::gc::GC_FLAG_FORWARDED == 0 + && shapes::object_shape_descriptor(obj) + .is_some_and(|shape| shape.object_kind == shapes::ShapeObjectKind::Ordinary) +} + +#[inline] +pub(crate) unsafe fn object_is_shaped(obj: *const ObjectHeader) -> bool { + if obj.is_null() { + return false; + } + let Some(header) = crate::value::addr_class::try_read_gc_header(obj as usize) else { + return false; + }; + header.obj_type == crate::gc::GC_TYPE_OBJECT + && header.gc_flags & crate::gc::GC_FLAG_FORWARDED == 0 +} + // #6812 spill lanes: the versioned write-loop emitter // (perry-codegen/src/stmt/loops.rs) addresses `meta.spill` at word 4 of the // ObjectMeta record and buffer elements one word past the ArrayHeader. Keep @@ -1771,7 +1803,7 @@ const _: () = assert!(std::mem::offset_of!(ObjectMeta, spill) == 32); const _: () = assert!(std::mem::size_of::() == 8); /// Fetch-or-allocate the per-object meta record. Caller must have already -/// established that `obj` is a live, non-RegExp `GC_TYPE_OBJECT` allocation +/// established that `obj` is a live `GC_TYPE_OBJECT` allocation /// (see `prototype_chain::meta_capable_object`). pub(crate) unsafe fn object_meta_ensure(obj: *mut ObjectHeader) -> *mut ObjectMeta { if !(*obj).meta.is_null() { @@ -1811,15 +1843,11 @@ pub(crate) unsafe fn object_meta_ensure(obj: *mut ObjectHeader) -> *mut ObjectMe meta } -/// GC slot accessor for the `meta` header edge (#6759 Phase B): a raw- -/// pointer child slot exactly like `gc_keys_array_slot`. Returns `None` for -/// a null meta AND for a `RegExpHeader` masquerading as `GC_TYPE_OBJECT` -/// (its bytes at this offset are native data — see the regex special case -/// in `gc_child_slots`). +/// GC slot accessor for the `meta` header edge (#6759 Phase B): a raw-pointer +/// child slot exactly like `gc_keys_array_slot`. The GC type table calls this +/// only for `GC_TYPE_OBJECT`; RegExp uses its dedicated slot descriptor. pub(crate) unsafe fn gc_object_meta_slot(user_ptr: usize) -> Option<*mut u64> { - if user_ptr == 0 - || crate::regex::regex_header_has_magic(user_ptr as *const crate::regex::RegExpHeader) - { + if user_ptr == 0 { return None; } let obj = user_ptr as *mut ObjectHeader; @@ -1841,21 +1869,22 @@ unsafe fn set_object_keys_array(obj: *mut ObjectHeader, keys_array: *mut ArrayHe // `is_shape_id` says so, for class instances too — and `clear_object_shape_stamp` // tests exactly that, so an instance still carrying its allocation-time // `parent_class_id` (never in the ShapeId range) is left alone. - if (*obj).keys_array != keys_array { - shapes::clear_object_shape_stamp(obj); - } - // #6893: the object's typed-shape layout descriptor is keyed by its - // keys_array (shared per shape via SHAPE_LAYOUTS). A keys_array pointer - // change is a shape change (add/delete key), so the exact typed layout no - // longer applies to THIS object. Pre-#6893 the per-object store-validation - // (`layout_note_slot`, keyed by the object address) caught this implicitly - // during the field shuffle; the shared descriptor lookup now misses on the - // NEW keys_array, so that trigger is lost — invalidate explicitly here. - // Gated: `mark_object_dynamic_shape_unknown` early-returns for objects that - // carry no typed layout, so plain/growing objects and initial construction - // (INTACT not yet set) pay nothing. - if (*obj).keys_array != keys_array { + let predecessor = shapes::object_shape_descriptor(obj); + let keys_changed = (*obj).keys_array != keys_array; + if keys_changed { + // #6893: the object's typed-shape layout descriptor is keyed by its + // keys_array (shared per shape via SHAPE_LAYOUTS). A pointer change + // makes that exact typed layout inapplicable. This is gated internally + // so plain/growing objects and initial construction pay nothing. + // + // Invalidate while the predecessor stamp is still authoritative. + // `layout_mark_unknown` reports the representation change through + // typed feedback, whose defensive shape lookup self-heals an + // unstamped object. Clearing first therefore let that re-entrant + // lookup publish an Ordinary descriptor for a class object; the + // structural synchronization below then inherited the wrong kind. mark_object_dynamic_shape_unknown(obj); + shapes::clear_object_shape_stamp(obj); } // GC_STORE_AUDIT(BARRIERED): keys_array pointer field is followed by an object-slot barrier. (*obj).keys_array = keys_array; @@ -1867,7 +1896,7 @@ unsafe fn set_object_keys_array(obj: *mut ObjectHeader, keys_array: *mut ArrayHe // #8067: the old header edge remains authoritative, but every visible // ShapeId must now resolve to the exact rooted ordered-keys/live-slot // descriptor. Same-pointer appends are versioned inside the helper. - shapes::synchronize_object_shape_descriptor(obj); + shapes::synchronize_object_shape_descriptor_from(obj, predecessor); } /// Publish a new authoritative live-inline-slot bound without ever exposing a @@ -1879,9 +1908,10 @@ unsafe fn set_object_keys_array(obj: *mut ObjectHeader, keys_array: *mut ArrayHe #[inline] pub(super) unsafe fn set_object_live_slot_count(obj: *mut ObjectHeader, field_count: u32) { if (*obj).field_count != field_count { + let predecessor = shapes::object_shape_descriptor(obj); shapes::clear_object_shape_stamp(obj); (*obj).field_count = field_count; - shapes::synchronize_object_shape_descriptor(obj); + shapes::synchronize_object_shape_descriptor_from(obj, predecessor); } else { shapes::debug_assert_object_shape_parity(obj); } @@ -1944,53 +1974,5 @@ pub(super) unsafe fn mark_object_dynamic_shape_unknown(obj: *mut ObjectHeader) { crate::gc::layout_mark_unknown(obj as *mut u8); } -pub(crate) unsafe fn gc_keys_array_slot(obj: *mut ObjectHeader) -> Option<*mut u64> { - if obj.is_null() || (*obj).keys_array.is_null() { - return None; - } - Some(&mut (*obj).keys_array as *mut _ as *mut u64) -} - -pub(crate) unsafe fn gc_field_slot_range( - obj: *mut ObjectHeader, -) -> Option { - if obj.is_null() { - return None; - } - let field_count = (*obj).field_count as usize; - if field_count > 1_000_000 { - return None; - } - let fields = (obj as *mut u8).add(std::mem::size_of::()) as *mut u64; - Some(crate::gc::HeapSlotRange::new(fields, field_count)) -} - -#[inline] -pub(super) unsafe fn rebuild_object_field_layout(obj: *mut ObjectHeader, slot_count: usize) { - let fields = (obj as *mut u8).add(std::mem::size_of::()) as *mut u64; - crate::gc::layout_rebuild_from_slots(obj as *mut u8, fields, slot_count); - if crate::arena::pointer_in_old_gen(obj as usize) { - for i in 0..slot_count { - let slot = fields.add(i); - crate::gc::runtime_write_barrier_slot(obj as usize, slot as usize, *slot); - } - } -} - -#[inline] -pub(super) unsafe fn rebuild_array_layout_from_slots(arr: *mut ArrayHeader) { - if arr.is_null() { - return; - } - let len = (*arr).length as usize; - let slots = (arr as *mut u8).add(std::mem::size_of::()) as *mut u64; - crate::gc::layout_rebuild_from_slots(arr as *mut u8, slots, len); - if crate::arena::pointer_in_old_gen(arr as usize) { - for i in 0..len { - let slot = slots.add(i); - crate::gc::runtime_write_barrier_slot(arr as usize, slot as usize, *slot); - } - } -} #[cfg(test)] mod tests; diff --git a/crates/perry-runtime/src/object/native_call_method.rs b/crates/perry-runtime/src/object/native_call_method.rs index 262e9eaa89..540187b395 100644 --- a/crates/perry-runtime/src/object/native_call_method.rs +++ b/crates/perry-runtime/src/object/native_call_method.rs @@ -60,9 +60,9 @@ pub(super) use typed_array::dispatch_typed_array_method; /// /// * the value is a NaN-boxed pointer to a real heap object above the handle /// band (excludes every small-handle registry receiver, and every primitive); -/// * its GC type is `GC_TYPE_OBJECT` and its `object_type` is -/// `OBJECT_TYPE_REGULAR` (excludes errors, arrays, maps, buffers, regexes, -/// closures — each of which the tower routes elsewhere); +/// * its GC type is `GC_TYPE_OBJECT` and its GcHeader carries no class-object +/// marker (excludes errors, arrays, maps, buffers, regexes, closures, and +/// class values — each of which the tower routes elsewhere); /// * `class_id` matches the cache key; /// * `meta` is null, so the object carries no `Object.setPrototypeOf` override, /// no per-key descriptor state, and no exotic-kind tag — this is *stricter* @@ -85,9 +85,9 @@ pub(super) use typed_array::dispatch_typed_array_method; /// * NaN-boxed pointer above the handle band — excludes every small-handle /// registry receiver (timers, sockets, zlib streams, TextDecoder, …) and /// every primitive; -/// * `GC_TYPE_OBJECT` + `OBJECT_TYPE_REGULAR` — excludes arrays, strings, -/// errors, maps, sets, regexes, closures, each of which the tower routes to -/// its own dispatcher; +/// * `GC_TYPE_OBJECT` without the class-object marker — excludes arrays, +/// strings, errors, maps, sets, regexes, closures, and class values, each of +/// which the tower routes to its own dispatcher; /// * not a registered `Buffer` and not a typed array — the two address-keyed /// probes the tower runs ahead of the class walk that a `GC_TYPE_OBJECT` /// receiver could in principle also answer. Both are latched (#7755), so in @@ -128,7 +128,7 @@ unsafe fn class_vtable_fast_guard(object: f64, method_bytes: &[u8]) -> Option<(u // so a `Some` here means both of those answer authoritatively from the meta // slot rather than falling back to a conservative `true`. let obj = super::prototype_chain::meta_capable_object(obj_addr)?; - if (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR { + if !crate::object::object_is_regular(obj) { return None; } // Null `meta` on a meta-capable object is what rules out BOTH a per-instance @@ -147,8 +147,10 @@ unsafe fn class_vtable_fast_guard(object: f64, method_bytes: &[u8]) -> Option<(u } // Own fields shadow vtable methods — same scan, same comparison, as the - // tower's field lookup. - let keys = (*obj).keys_array; + // tower's field lookup. ShapeId supplies both the moving root and its exact + // logical length; the ObjectHeader mirrors are compatibility scratch only. + let descriptor = crate::object::shapes::object_shape_descriptor(obj)?; + let keys = descriptor.keys as usize as *mut ArrayHeader; if !keys.is_null() { let keys_ptr = keys as usize; // Band predicate, not a bare floor (#7531/#7709): the 0x10000 floor this @@ -158,7 +160,7 @@ unsafe fn class_vtable_fast_guard(object: f64, method_bytes: &[u8]) -> Option<(u { return None; } - let key_count = crate::array::js_array_length(keys) as usize; + let key_count = descriptor.logical_key_count as usize; if key_count > 65536 { return None; } @@ -183,10 +185,10 @@ unsafe fn class_vtable_fast_guard(object: f64, method_bytes: &[u8]) -> Option<(u /// [`class_vtable_fast_guard`] does not pin. /// /// * the `using` / `await using` disposal hooks read a SYMBOL-keyed own -/// property (`obj[Symbol.dispose]`), which the guard's string-`keys_array` -/// scan cannot see: two instances of one class can differ, so a resolution -/// cached from an instance without the symbol would route a later instance -/// with one straight past its custom disposer; +/// property (`obj[Symbol.dispose]`), which the guard's descriptor-backed +/// string-key scan cannot see: two instances of one class can differ, so a +/// resolution cached from an instance without the symbol would route a later +/// instance with one straight past its custom disposer; /// * the iterator helpers (`map`/`filter`/`take`/…) dispatch on whether the /// receiver *is* an iterator. #[inline] @@ -931,9 +933,7 @@ unsafe fn gc_pointer_and_type_from_value(value: f64) -> Option<(*const u8, u8)> // // * `set::is_registered_set` ends in `obj_type == GC_TYPE_SET`; // * `map::is_registered_map` ends in `obj_type == GC_TYPE_MAP`; - // * `regex::is_regex_pointer` matches the header magic of a - // `gc_malloc(_, GC_TYPE_OBJECT)` allocation, and the sole - // `REGEX_POINTERS` insert (`js_regexp_new`) allocates exactly that; + // * RegExp has the dedicated `GC_TYPE_REGEXP` kind; // * a `Symbol` of any storage carries `SYMBOL_MAGIC` in its first word. // // The one kind the header cannot speak for is the `Box`-leaked symbol @@ -953,7 +953,7 @@ unsafe fn gc_pointer_and_type_from_value(value: f64) -> Option<(*const u8, u8)> let excluded = match obj_type { crate::gc::GC_TYPE_SET => crate::set::is_registered_set(addr), crate::gc::GC_TYPE_MAP => crate::map::is_registered_map(addr), - crate::gc::GC_TYPE_OBJECT => crate::regex::is_regex_pointer(ptr as *const u8), + crate::gc::GC_TYPE_REGEXP => true, _ => false, }; if excluded { @@ -1780,8 +1780,6 @@ pub unsafe extern "C" fn js_native_call_method( } if gc_type != crate::gc::GC_TYPE_OBJECT { - // Only accept object_type == 1 (OBJECT_TYPE_REGULAR) - let object_type = (*obj).object_type; // Closes #645: when a method falls through every dispatcher // and returns NULL_OBJECT_BYTES (e.g. drizzle's // `this.client.prepare(...)` where `this.client` resolved to @@ -1805,13 +1803,15 @@ pub unsafe extern "C" fn js_native_call_method( let null_obj_ptr = &NULL_OBJECT_BYTES as *const NullObjectBytes as *mut u8; return f64::from_bits(JSValue::pointer(null_obj_ptr).bits()); } - if object_type != crate::error::OBJECT_TYPE_REGULAR { - let null_obj_ptr = &NULL_OBJECT_BYTES as *const NullObjectBytes as *mut u8; - return f64::from_bits(JSValue::pointer(null_obj_ptr).bits()); - } + let null_obj_ptr = &NULL_OBJECT_BYTES as *const NullObjectBytes as *mut u8; + return f64::from_bits(JSValue::pointer(null_obj_ptr).bits()); } - let keys = (*obj).keys_array; + let Some(descriptor) = crate::object::shapes::object_shape_descriptor(obj) else { + let null_obj_ptr = &NULL_OBJECT_BYTES as *const NullObjectBytes as *mut u8; + return f64::from_bits(JSValue::pointer(null_obj_ptr).bits()); + }; + let keys = descriptor.keys as usize as *mut ArrayHeader; if !keys.is_null() { // Validate keys_array pointer before dereferencing @@ -1827,7 +1827,7 @@ pub unsafe extern "C" fn js_native_call_method( // GcHeader-based validation. // Search for the method in the object's fields - let key_count = crate::array::js_array_length(keys) as usize; + let key_count = descriptor.logical_key_count as usize; // Sanity check key_count if key_count > 65536 { let null_obj_ptr = &NULL_OBJECT_BYTES as *const NullObjectBytes as *mut u8; diff --git a/crates/perry-runtime/src/object/native_call_method/object_proto.rs b/crates/perry-runtime/src/object/native_call_method/object_proto.rs index 48d06bb042..71460c4a45 100644 --- a/crates/perry-runtime/src/object/native_call_method/object_proto.rs +++ b/crates/perry-runtime/src/object/native_call_method/object_proto.rs @@ -223,8 +223,8 @@ pub(crate) unsafe fn js_object_is_prototype_of_value(receiver: f64, target: f64) } // A RegExp's `[[Prototype]]` chain is `RegExp.prototype → Object.prototype`. - // The RegExpHeader isn't a plain GC_TYPE_OBJECT with a registered class - // prototype, so the generic class-id walk below misses it (which is why + // The dedicated RegExp cell has no registered class prototype, so the + // generic class-id walk below misses it (which is why // `RegExp.prototype.isPrototypeOf(re)` returned false). Handle it directly. { let tv = JSValue::from_bits(target.to_bits()); diff --git a/crates/perry-runtime/src/object/native_call_method/probe_dispatch_tests.rs b/crates/perry-runtime/src/object/native_call_method/probe_dispatch_tests.rs index 876d162c6b..5838c2d9e5 100644 --- a/crates/perry-runtime/src/object/native_call_method/probe_dispatch_tests.rs +++ b/crates/perry-runtime/src/object/native_call_method/probe_dispatch_tests.rs @@ -157,8 +157,8 @@ fn exotic_receivers_are_still_excluded() { ); } -/// RegExp is the one exotic kind that genuinely IS a `GC_TYPE_OBJECT` -/// allocation, so it is the reason the GC_TYPE_OBJECT arm still probes. +/// RegExp has its own GC kind, so it must be rejected without entering the +/// ordinary-object arm or consulting an `ObjectHeader` payload word. #[cfg(feature = "regex-engine")] #[test] fn regexp_receiver_is_still_excluded() { @@ -168,8 +168,8 @@ fn regexp_receiver_is_still_excluded() { assert!(re != 0, "test premise: RegExp allocated"); assert!( classify(re).is_none(), - "a RegExp is a GC_TYPE_OBJECT allocation and must still be excluded by \ - the regex probe the GC_TYPE_OBJECT arm keeps" + "a RegExp has GC_TYPE_REGEXP and must be excluded before ordinary-object \ + header reads" ); } @@ -252,7 +252,7 @@ fn the_magic_screen_covers_every_symbol_and_no_ordinary_object() { let excluded_without_the_screen = match obj_type { crate::gc::GC_TYPE_SET => crate::set::is_registered_set(sym), crate::gc::GC_TYPE_MAP => crate::map::is_registered_map(sym), - crate::gc::GC_TYPE_OBJECT => crate::regex::is_regex_pointer(sym as *const u8), + crate::gc::GC_TYPE_REGEXP => true, _ => false, }; assert!( diff --git a/crates/perry-runtime/src/object/prototype_chain.rs b/crates/perry-runtime/src/object/prototype_chain.rs index 23b8f169d8..7dd6885a43 100644 --- a/crates/perry-runtime/src/object/prototype_chain.rs +++ b/crates/perry-runtime/src/object/prototype_chain.rs @@ -124,10 +124,9 @@ fn get_object_prototypes() -> &'static Mutex> { /// #6759 Phase B: classify `obj_ptr` as a genuine shaped `GC_TYPE_OBJECT` /// whose header can carry the per-object meta record. Everything else — -/// arrays, typed arrays, native handle-band ids, proxy ids, and the -/// `RegExpHeader` that is tagged `GC_TYPE_OBJECT` but has a different -/// layout — returns `None` and stays on the residual registry. The -/// classification is a pure function of the allocation, so an owner is +/// arrays, typed arrays, native handle-band ids, proxy ids, and the dedicated +/// `GC_TYPE_REGEXP` cell — returns `None` and stays on the residual registry. +/// The classification is a pure function of the allocation, so an owner is /// always on exactly one of the two storages. pub(crate) unsafe fn meta_capable_object(obj_ptr: usize) -> Option<*mut crate::ObjectHeader> { if !crate::value::addr_class::is_above_handle_band(obj_ptr) @@ -139,9 +138,6 @@ pub(crate) unsafe fn meta_capable_object(obj_ptr: usize) -> Option<*mut crate::O if header.obj_type != crate::gc::GC_TYPE_OBJECT { return None; } - if crate::regex::regex_header_has_magic(obj_ptr as *const crate::regex::RegExpHeader) { - return None; - } Some(obj_ptr as *mut crate::ObjectHeader) } @@ -201,7 +197,14 @@ fn object_set_static_prototype_impl(obj_ptr: usize, proto_bits: u64, instance_ov // registry. unsafe { if let Some(obj) = meta_capable_object(obj_ptr) { - let meta = crate::object::object_meta_ensure(obj); + // `object_meta_ensure` allocates and may evacuate the owner. Keep + // the caller's pointer rooted and reload it before the semantic + // ShapeId transition below. + let scope = crate::gc::RuntimeHandleScope::new(); + let obj_handle = scope.root_raw_mut_ptr(obj); + let (meta, obj) = obj_handle.across_mut::(|| { + crate::object::object_meta_ensure(obj) + }); (*meta).prototype = proto_bits; if instance_override { (*meta).flags |= crate::object::OBJECT_META_FLAG_PROTO_OVERRIDE; @@ -214,6 +217,9 @@ fn object_set_static_prototype_impl(obj_ptr: usize, proto_bits: u64, instance_ov &(*meta).prototype as *const u64 as usize, proto_bits, ); + if instance_override { + crate::object::shapes::transition_object_shape_semantics(obj); + } return; } } diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index b9ba97945a..430e747032 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -14,13 +14,11 @@ //! re-validates the key bytes. Separately, every published `ShapeId` resolves //! in this agent's `RuntimeState` to an immutable descriptor containing the //! ordered-keys edge plus the exact logical-key and live-inline-slot bounds. -//! The descriptor table is weak: a live object's authoritative header edge -//! keeps keys alive and synchronizes the descriptor mirror, while dead-key -//! entries are pruned after tracing. This avoids -//! turning historical shapes into permanent roots. `ObjectHeader::{keys_array, -//! field_count}` remain authoritative in this first slice; publication helpers -//! assert descriptor parity and every old guard stays in place as redundant -//! evidence. +//! The descriptor table is agent-local while ids are process-global. A live +//! object's ShapeId is authoritative for its ordered keys, logical-key count, +//! live inline-slot bound, and semantic generation. The legacy +//! `ObjectHeader::{keys_array,field_count}` words remain ABI mirrors until +//! #8047 removes them; guards and GC must not use their values as shape facts. use crate::array::ArrayHeader; use std::cell::RefCell; @@ -47,6 +45,20 @@ pub(crate) struct ShapeDescriptor { pub(crate) keys: u64, pub(crate) logical_key_count: u32, pub(crate) live_inline_slot_count: u32, + /// Zero for ordinary structural shapes. Descriptor/prototype mutations + /// mint a process-unique nonzero generation so two semantically different + /// layouts can never compare equal merely because their keys/counts do. + pub(crate) semantic_generation: u64, + /// Semantic receiver kind carried by this exact ShapeId. This is kept in + /// the authoritative descriptor rather than `GcHeader::_reserved`, whose + /// bits belong to the GC layout/age protocol and object feature flags. + pub(crate) object_kind: ShapeObjectKind, +} + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub(crate) enum ShapeObjectKind { + Ordinary, + Class, } #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] @@ -54,12 +66,16 @@ struct ShapeFacts { keys: u64, logical_key_count: u32, live_inline_slot_count: u32, + semantic_generation: u64, + object_kind: ShapeObjectKind, } struct ShapeTableInner { indices: crate::fast_hash::PtrHashMap, descriptors: HashMap, - ids_by_facts: HashMap, + /// Exact-facts reverse index. More than one id is legal when a worker + /// minted a local descriptor before a process-global module id arrived. + ids_by_facts: HashMap>, /// Keys-array address -> every descriptor id that currently names it. /// Same-address key-count retirement uses this index instead of scanning /// every shape ever observed by the agent. @@ -89,6 +105,8 @@ fn descriptor_facts(descriptor: ShapeDescriptor) -> ShapeFacts { keys: descriptor.keys, logical_key_count: descriptor.logical_key_count, live_inline_slot_count: descriptor.live_inline_slot_count, + semantic_generation: descriptor.semantic_generation, + object_kind: descriptor.object_kind, } } @@ -104,11 +122,27 @@ fn remove_id_from_keys_index(inner: &mut ShapeTableInner, keys: u64, id: u32) { } } +fn remove_id_from_facts_index(inner: &mut ShapeTableInner, facts: ShapeFacts, id: u32) { + let remove_entry = if let Some(ids) = inner.ids_by_facts.get_mut(&facts) { + ids.retain(|&candidate| candidate != id); + ids.is_empty() + } else { + false + }; + if remove_entry { + inner.ids_by_facts.remove(&facts); + } +} + fn rebuild_descriptor_reverse_indices(inner: &mut ShapeTableInner) { - let mut ids_by_facts = HashMap::with_capacity(inner.descriptors.len()); + let mut ids_by_facts: HashMap> = + HashMap::with_capacity(inner.descriptors.len()); let mut ids_by_keys: HashMap> = HashMap::new(); for (&id, &descriptor) in &inner.descriptors { - ids_by_facts.insert(descriptor_facts(descriptor), id); + ids_by_facts + .entry(descriptor_facts(descriptor)) + .or_default() + .push(id); ids_by_keys.entry(descriptor.keys).or_default().push(id); } inner.ids_by_facts = ids_by_facts; @@ -134,6 +168,8 @@ pub(crate) const SHAPE_ID_END: u32 = 0xC000_0000; static SHAPE_ID_NEXT: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(SHAPE_ID_BASE); +static SHAPE_SEMANTIC_NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); + #[inline] pub(crate) fn is_shape_id(v: u32) -> bool { (SHAPE_ID_BASE..SHAPE_ID_END).contains(&v) @@ -148,9 +184,8 @@ pub(crate) fn is_shape_id_token(v: usize) -> bool { v >= SHAPE_ID_BASE as usize && v < SHAPE_ID_END as usize } -/// #6804: lifts a ShapeId into the per-site PIC token space, ABOVE the -/// 48-bit pointer range, so an id token can never numerically equal a -/// keys-array pointer token. MUST match the literal the PIC IR emits in +/// Lifts a ShapeId into the per-site PIC token space. MUST match the literal +/// the PIC IR emits in /// `perry-codegen/src/expr/property_get/generic_dispatch.rs` /// (4611686018427387904 = 1 << 62). pub(crate) const PIC_ID_TOKEN_BIT: u64 = 1 << 62; @@ -158,6 +193,12 @@ pub(crate) const PIC_ID_TOKEN_BIT: u64 = 1 << 62; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) struct ShapeIdExhausted; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ShapeDescriptorError { + IdExhausted, + InvalidFacts, +} + fn alloc_shape_id_from(next: &std::sync::atomic::AtomicU32) -> Result { use std::sync::atomic::Ordering; loop { @@ -181,47 +222,101 @@ fn alloc_shape_id() -> Result { alloc_shape_id_from(&SHAPE_ID_NEXT) } -/// Get or create the exact descriptor. Exhaustion is recoverable and -/// fail-closed: callers leave the object unstamped and continue through the -/// retained authoritative header pointer/count guards. No id is reused and no -/// descriptor lookup can alias. -pub(crate) fn shape_descriptor_ensure( +/// Get or create the exact structural descriptor. The public allocation and +/// mutation paths turn exhaustion into a fail-stop before publishing an +/// untracked layout; the `Result` stays explicit so the allocator boundary and +/// its exhaustion tests remain reviewable. +fn shape_descriptor_ensure_with_generation( keys: *const ArrayHeader, logical_key_count: u32, live_inline_slot_count: u32, -) -> Result { + semantic_generation: u64, + object_kind: ShapeObjectKind, +) -> Result { let keys_id = keys as usize; - if keys_id == 0 { - return Err(ShapeIdExhausted); + if keys_id == 0 && logical_key_count != 0 { + return Err(ShapeDescriptorError::InvalidFacts); } let facts = ShapeFacts { keys: keys_id as u64, logical_key_count, live_inline_slot_count, + semantic_generation, + object_kind, }; let mut inner = crate::state::state().shapes.inner.borrow_mut(); - if let Some(&id) = inner.ids_by_facts.get(&facts) { + if let Some(id) = inner + .ids_by_facts + .get(&facts) + .and_then(|ids| ids.first().copied()) + { return Ok(id); } - let id = alloc_shape_id()?; + let id = alloc_shape_id().map_err(|_| ShapeDescriptorError::IdExhausted)?; let descriptor = ShapeDescriptor { keys: keys_id as u64, logical_key_count, live_inline_slot_count, + semantic_generation, + object_kind, }; // Publish by-id first, then the reverse accelerator. An ObjectHeader is // stamped only after this function returns, so a visible id always has a // complete descriptor. inner.descriptors.insert(id, descriptor); - inner.ids_by_facts.insert(facts, id); + inner.ids_by_facts.entry(facts).or_default().push(id); inner.ids_by_keys.entry(facts.keys).or_default().push(id); Ok(id) } +pub(crate) fn shape_descriptor_ensure( + keys: *const ArrayHeader, + logical_key_count: u32, + live_inline_slot_count: u32, +) -> Result { + shape_descriptor_ensure_with_generation( + keys, + logical_key_count, + live_inline_slot_count, + 0, + ShapeObjectKind::Ordinary, + ) +} + +#[cold] +#[inline(never)] +fn shape_id_exhausted_abort() -> ! { + eprintln!("Perry ShapeId space exhausted; refusing to publish an untracked object shape"); + std::process::abort() +} + +#[cold] +#[inline(never)] +fn invalid_shape_facts_abort() -> ! { + eprintln!("Perry internal error: refusing to publish invalid object shape facts"); + std::process::abort() +} + +#[inline] +fn shape_descriptor_error_abort(error: ShapeDescriptorError) -> ! { + match error { + ShapeDescriptorError::IdExhausted => shape_id_exhausted_abort(), + ShapeDescriptorError::InvalidFacts => invalid_shape_facts_abort(), + } +} + +#[inline] +fn publish_shape_result(result: Result) -> u32 { + match result { + Ok(id) => id, + Err(error) => shape_descriptor_error_abort(error), + } +} + /// Compatibility mint for canonical shapes whose key and live-slot counts are /// identical. New object-aware paths use [`shape_descriptor_ensure`] directly. pub(crate) fn shape_id_for_keys_ensure(keys: *const ArrayHeader, key_count: u32) -> u32 { - shape_descriptor_ensure(keys, key_count, key_count).unwrap_or(0) + publish_shape_result(shape_descriptor_ensure(keys, key_count, key_count)) } pub(crate) fn shape_descriptor_by_id(shape_id: u32) -> Option { @@ -249,54 +344,73 @@ pub extern "C" fn js_object_shape_id_for_keys(keys: u64, key_count: u32) -> u32 shape_id_for_keys_ensure(keys as usize as *const ArrayHeader, key_count) } +/// Install a process-global id into this agent's local descriptor table. +/// Module globals are initialized once per process, while workers own distinct +/// runtime state and moving keys pointers. Global id uniqueness makes a local +/// first installation unambiguous; an existing different descriptor fails +/// closed and the caller mints a fresh local id instead. +fn install_external_shape_id( + id: u32, + keys: *const ArrayHeader, + logical_key_count: u32, + live_inline_slot_count: u32, +) -> bool { + if !is_shape_id(id) || (keys.is_null() && logical_key_count != 0) { + return false; + } + let descriptor = ShapeDescriptor { + keys: keys as usize as u64, + logical_key_count, + live_inline_slot_count, + semantic_generation: 0, + object_kind: ShapeObjectKind::Ordinary, + }; + let facts = descriptor_facts(descriptor); + let mut inner = crate::state::state().shapes.inner.borrow_mut(); + if let Some(existing) = inner.descriptors.get(&id) { + return *existing == descriptor; + } + // A worker can have minted an equivalent local descriptor before module + // initialization installs the process-global codegen id. Keep both id + // descriptors valid for already-published objects and make the external + // id canonical for subsequent births in this agent. + inner.descriptors.insert(id, descriptor); + // An equivalent local descriptor can predate module initialization. Keep + // both reverse-index entries and prefer the external id for subsequent + // births in this agent; already-published local ids remain resolvable. + inner.ids_by_facts.entry(facts).or_default().insert(0, id); + inner + .ids_by_keys + .entry(descriptor.keys) + .or_default() + .push(id); + true +} + // --------------------------------------------------------------------------- -// #6759 C3 rung 1 — THE SHAPE WORD IS UNIFORM. +// #8067 — THE SHAPE WORD IS UNIFORM AND AUTHORITATIVE. // -// `ObjectHeader.parent_class_id` IS the shape word. Before this rung the stamp -// was additionally gated on `class_id == 0`, so a CLASS INSTANCE had no shape -// word at all: its only header evidence of a key-set change was the -// `keys_array` POINTER, which is exactly why `class_field_inline_guard` -// compares that pointer, and why the #7916 header shrink could not delete it. +// `ObjectHeader.parent_class_id` is the shape word. Every shaped object is +// birth-stamped; inheritance lives in the class-id-keyed registry instead. // // The gate is gone. The rule is now, for every receiver kind: // // the word is a ShapeId <=> is_shape_id(word) // -// which is already what all three emitted PICs test — they discriminate on the -// range alone (`property_get/generic_dispatch.rs`, `expr/proxy_reflect.rs` ×2), -// never on `class_id`. So relaxing the runtime gates makes the runtime agree -// with the IR rather than introducing a new mode. -// -// This is only sound because rung 0 (#7981) removed the LAST reader of the -// header word as inheritance data (`thread.rs::serialize_object` now takes the -// parent edge from the class-id-keyed registry). Every parent-chain walk in the -// tree goes through `get_parent_class_id(class_id)`; nothing reads the header -// word for inheritance. Overwriting a class instance's `parent_class_id` with a -// ShapeId therefore loses no information — the registry still has the edge, and -// it was registered from a compile-time constant either by the allocator or by -// the `js_register_class_parent` module-init prelude. -// -// The stamp is LAZY: a class instance is stamped at its first by-name resolve, -// so a freshly `new`'d instance still reads as unstamped and falls back to the -// keys-pointer token. Eager birth stamping in codegen is rung 2. +// which is exactly what emitted PICs test: the ShapeId range and value, never a +// moving keys address or an ObjectHeader compatibility mirror. // --------------------------------------------------------------------------- /// True when `obj` really is an `ObjectHeader` whose word 2 may be written. /// -/// A `RegExpHeader` aliases `GC_TYPE_OBJECT` with a DIFFERENT layout — its -/// offset 8 is the low half of `pattern_ptr`, and offset 4 is the high half of -/// `regex_ptr`, which reads as `class_id == 0` on every 48-bit-address target. -/// So the old `class_id == 0` gate never excluded a RegExp either; two of the -/// four mint sites carried an explicit magic check and two did not. Routing -/// every site through this predicate closes that gap in the same edit that -/// removes the `class_id` discriminant. +/// RegExp now has a distinct GC kind, so ShapeId publication never needs to +/// inspect an ObjectHeader payload word to distinguish it. #[inline] pub(crate) unsafe fn shape_word_is_writable(obj: *const crate::object::ObjectHeader) -> bool { - !crate::regex::regex_header_has_magic(obj as *const crate::regex::RegExpHeader) + crate::object::object_is_shaped(obj) } -/// The receiver's ShapeId, or 0 when it carries none (unstamped, or the word -/// still holds inheritance data left over from allocation). +/// The receiver's ShapeId, or 0 when it is not a shaped object. #[inline] pub(crate) unsafe fn object_shape_stamp(obj: *const crate::object::ObjectHeader) -> u32 { let word = (*obj).parent_class_id; @@ -308,9 +422,9 @@ pub(crate) unsafe fn object_shape_stamp(obj: *const crate::object::ObjectHeader) } /// Stamp `obj` with the exact ShapeId of `keys`, minting the descriptor on -/// first touch. Returns the id, or 0 when the receiver is not stampable (a -/// RegExp alias) or the id range is exhausted. Exhaustion leaves the object -/// unstamped so the retained header pointer/count checks remain authoritative. +/// first touch. Returns 0 only when the receiver is not a shaped object. +/// Exhaustion fails stop: no live object may depend on the +/// compatibility pointer/count mirrors for its shape. #[inline] pub(crate) unsafe fn stamp_object_shape( obj: *mut crate::object::ObjectHeader, @@ -320,10 +434,20 @@ pub(crate) unsafe fn stamp_object_shape( if !shape_word_is_writable(obj) { return 0; } - let Ok(id) = shape_descriptor_ensure(keys, key_count, (*obj).field_count) else { - clear_object_shape_stamp(obj); - return 0; + let Some(lineage) = object_shape_descriptor(obj) else { + let id = shape_descriptor_ensure(keys, key_count, (*obj).field_count) + .unwrap_or_else(|error| shape_descriptor_error_abort(error)); + (*obj).parent_class_id = id; + debug_assert_object_shape_parity(obj); + return id; }; + let id = publish_shape_result(shape_descriptor_ensure_with_generation( + keys, + key_count, + lineage.live_inline_slot_count, + lineage.semantic_generation, + lineage.object_kind, + )); (*obj).parent_class_id = id; debug_assert_object_shape_parity(obj); id @@ -331,36 +455,13 @@ pub(crate) unsafe fn stamp_object_shape( /// Birth-stamp a NEWBORN receiver with an already-minted ShapeId after checking /// its descriptor against the completed header. A missing, foreign, or -/// count-mismatched id is replaced with an exact local descriptor when ids -/// remain available; exhaustion leaves the receiver explicitly unstamped. +/// count-mismatched id is replaced with an exact local descriptor. A valid +/// process-global id absent from this worker is installed with the worker's +/// local moving keys pointer before it is stamped. /// -/// ★ **A shape's population must be UNIFORMLY stamped or uniformly not.** The -/// emitted read PIC derives its ENTIRE cache token from this word -/// (`perry-codegen/src/expr/property_get/generic_dispatch.rs`): -/// -/// ```text -/// is_stamp = (parent_class_id - 0x8000_0000) u< 0x4000_0000 -/// token = is_stamp ? (parent_class_id | 1<<62) : keys_array -/// ``` -/// -/// so a stamped receiver and an unstamped one OF THE SAME SHAPE compute two -/// DIFFERENT tokens, and a site that sees both can never hold a hit. It is not -/// a slow start — it is a permanent 0% hit rate: instance #1 misses, is -/// stamped, primes the id token; instance #2 is newborn, computes the -/// keys-pointer token, misses; the handler re-primes the same id; instance #3 -/// misses. Forever. -/// -/// #6759 C3 rung 1 (#7983) stamped class instances only LAZILY, at the first -/// by-name resolve, and that is exactly what it cost — measured in -/// instructions retired, isolated against its own parent: `cycles` +54.3%, -/// `deeplist` +45.2%, `interp` +28.3%, `pipeline` +23.9%, `iso_miss` +22.9%, -/// while the object-literal benchmarks (`churn` +1.2%, `retain` +0.2%) and -/// `fib40` (+0.04%) did not move — literals have been birth-stamped since -/// #6804, so their population was always uniform. -/// -/// Rung 2 (#8009) closed the compiled path. **Every OTHER allocator that -/// installs a shape-cached keys array on a fresh `ObjectHeader` must call this -/// too**, or its classes keep the split. +/// Every allocator that installs a shape-cached keys array on a fresh +/// `ObjectHeader` must call this so all runtime and emitted guards observe the +/// same descriptor identity from birth. /// /// No `shape_word_is_writable` check: the callers have just written /// `object_type`/`class_id` into a header they allocated, so the receiver is a @@ -370,7 +471,18 @@ pub(crate) unsafe fn birth_stamp_object_shape( obj: *mut crate::object::ObjectHeader, runtime_shape_id: u32, ) { - if is_shape_id(runtime_shape_id) && descriptor_matches_object(runtime_shape_id, obj) { + if obj.is_null() || !shape_word_is_writable(obj) { + return; + } + let current = object_shape_descriptor(obj).unwrap_or_else(|| { + synchronize_object_shape_descriptor(obj); + object_shape_descriptor(obj).expect("shape synchronization must publish a descriptor") + }); + let keys = current.keys as usize as *mut ArrayHeader; + let key_count = current.logical_key_count; + let supplied_id_is_local = descriptor_matches_object(runtime_shape_id, obj) + || install_external_shape_id(runtime_shape_id, keys, key_count, (*obj).field_count); + if supplied_id_is_local { (*obj).parent_class_id = runtime_shape_id; debug_assert_object_shape_parity(obj); } else { @@ -379,20 +491,31 @@ pub(crate) unsafe fn birth_stamp_object_shape( } /// Install the exact descriptor for the object's current authoritative header -/// facts. This is the only shape publication operation used by mutations. -/// Exhaustion (or no keys) clears the stamp, retaining the old exact guards. +/// facts. This is the only structural shape publication operation used by +/// mutations. Keyless objects receive a descriptor too. pub(crate) unsafe fn synchronize_object_shape_descriptor( obj: *mut crate::object::ObjectHeader, +) -> u32 { + let predecessor = object_shape_descriptor(obj); + synchronize_object_shape_descriptor_from(obj, predecessor) +} + +/// Structural synchronization after a caller has temporarily cleared the +/// stamp. `predecessor` carries semantic lineage (including class kind) across +/// the pointer/count mutation without exposing stale structural facts. +pub(crate) unsafe fn synchronize_object_shape_descriptor_from( + obj: *mut crate::object::ObjectHeader, + predecessor: Option, ) -> u32 { if obj.is_null() || !shape_word_is_writable(obj) { return 0; } let keys = (*obj).keys_array; - if keys.is_null() { - clear_object_shape_stamp(obj); - return 0; - } - let key_count = crate::array::keys_array_len_capped_to_capacity(keys) as u32; + let key_count = if keys.is_null() { + 0 + } else { + crate::array::keys_array_len_capped_to_capacity(keys) as u32 + }; // A same-address length change is legal only for an owned keys array. A // shared array must have cloned before push; otherwise siblings already @@ -418,20 +541,114 @@ pub(crate) unsafe fn synchronize_object_shape_descriptor( clear_object_shape_stamp(obj); return 0; } - retire_key_count_versions(keys as u64, key_count); + retain_key_count_versions(keys as u64); } } - let Ok(id) = shape_descriptor_ensure(keys, key_count, (*obj).field_count) else { - clear_object_shape_stamp(obj); + // A caller-supplied predecessor was captured before it temporarily + // cleared the stamp to mutate structural facts, so it is the semantic + // authority for this transition. A re-entrant observer can defensively + // self-heal the zero stamp in that window; never let that interim + // descriptor replace the saved class/semantic lineage. + let lineage = predecessor.or_else(|| shape_descriptor_by_id(old_id)); + let semantic_generation = lineage + .map(|descriptor| descriptor.semantic_generation) + .unwrap_or(0); + let object_kind = lineage + .map(|descriptor| descriptor.object_kind) + .unwrap_or(ShapeObjectKind::Ordinary); + let id = publish_shape_result(shape_descriptor_ensure_with_generation( + keys, + key_count, + (*obj).field_count, + semantic_generation, + object_kind, + )); + (*obj).parent_class_id = id; + debug_assert_object_shape_parity(obj); + id +} + +/// Mint an exact successor for a descriptor/prototype semantic transition. +/// The structural facts remain unchanged, but the process-unique generation +/// prevents a cache trained before the transition from comparing equal after +/// it. Shared siblings retain their immutable predecessor descriptor. +pub(crate) unsafe fn transition_object_shape_semantics( + obj: *mut crate::object::ObjectHeader, +) -> u32 { + if obj.is_null() || !shape_word_is_writable(obj) { return 0; - }; + } + let current = object_shape_descriptor(obj).unwrap_or_else(|| { + synchronize_object_shape_descriptor(obj); + object_shape_descriptor(obj).expect("shape synchronization must publish a descriptor") + }); + let keys = current.keys as usize as *mut ArrayHeader; + let key_count = current.logical_key_count; + let generation = SHAPE_SEMANTIC_NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if generation == 0 { + shape_id_exhausted_abort(); + } + let id = publish_shape_result(shape_descriptor_ensure_with_generation( + keys, + key_count, + current.live_inline_slot_count, + generation, + current.object_kind, + )); + (*obj).parent_class_id = id; + debug_assert_object_shape_parity(obj); + id +} + +/// Turn a class-expression object into a class receiver. The kind is part of +/// the exact immutable descriptor, so it cannot alias GC layout bits and every +/// pre-mark ShapeId guard permanently misses afterward. +pub(crate) unsafe fn transition_object_shape_to_class( + obj: *mut crate::object::ObjectHeader, +) -> u32 { + if obj.is_null() || !shape_word_is_writable(obj) { + return 0; + } + let current = object_shape_descriptor(obj).unwrap_or_else(|| { + synchronize_object_shape_descriptor(obj); + object_shape_descriptor(obj).expect("shape synchronization must publish a descriptor") + }); + if current.object_kind == ShapeObjectKind::Class { + return object_shape_stamp(obj); + } + let generation = SHAPE_SEMANTIC_NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if generation == 0 { + shape_id_exhausted_abort(); + } + let id = publish_shape_result(shape_descriptor_ensure_with_generation( + current.keys as usize as *const ArrayHeader, + current.logical_key_count, + current.live_inline_slot_count, + generation, + ShapeObjectKind::Class, + )); (*obj).parent_class_id = id; debug_assert_object_shape_parity(obj); id } -fn retire_key_count_versions(keys: u64, current_key_count: u32) { +/// Authoritative descriptor for a genuine shaped object. +#[inline] +pub(crate) unsafe fn object_shape_descriptor( + obj: *const crate::object::ObjectHeader, +) -> Option { + shape_descriptor_by_id(object_shape_stamp(obj)) +} + +#[inline] +pub(crate) unsafe fn object_shape_id(obj: *const crate::object::ObjectHeader) -> u32 { + object_shape_descriptor(obj) + .map(|_| object_shape_stamp(obj)) + .unwrap_or(0) +} + +fn retain_key_count_versions(keys: u64) { let mut inner = crate::state::state().shapes.inner.borrow_mut(); let Some(ids) = inner.ids_by_keys.remove(&keys) else { return; @@ -450,10 +667,12 @@ fn retire_key_count_versions(keys: u64, current_key_count: u32) { if !correct_ids.contains(&id) { correct_ids.push(id); } - } else if descriptor.logical_key_count != current_key_count { - inner.descriptors.remove(&id); - inner.ids_by_facts.remove(&descriptor_facts(descriptor)); } else { + // Keep immutable historical descriptors addressable by id. An + // append under an owned keys allocation preserves the old prefix, + // and a stale cache/object may still carry either a local or an + // equivalent external id. Dead-key pruning reclaims the whole + // lineage once no live owner reaches the keys allocation. current_ids.push(id); } } @@ -468,9 +687,13 @@ fn descriptor_matches_object(shape_id: u32, obj: *const crate::object::ObjectHea }; unsafe { let keys = (*obj).keys_array; - !keys.is_null() - && d.keys == keys as u64 - && d.logical_key_count == crate::array::keys_array_len_capped_to_capacity(keys) as u32 + let key_count = if keys.is_null() { + 0 + } else { + crate::array::keys_array_len_capped_to_capacity(keys) as u32 + }; + d.keys == keys as u64 + && d.logical_key_count == key_count && d.live_inline_slot_count == (*obj).field_count } } @@ -533,8 +756,12 @@ pub(crate) unsafe fn synchronize_live_object_shape_descriptor_after_header_visit (old_facts, descriptor_facts(*descriptor)) }; if new_facts != old_facts { - inner.ids_by_facts.remove(&old_facts); - inner.ids_by_facts.insert(new_facts, shape_id); + remove_id_from_facts_index(&mut inner, old_facts, shape_id); + inner + .ids_by_facts + .entry(new_facts) + .or_default() + .push(shape_id); remove_id_from_keys_index(&mut inner, old_facts.keys, shape_id); inner .ids_by_keys @@ -798,7 +1025,7 @@ pub(crate) fn test_drop_shape_descriptors(keys_id: usize) { .unwrap_or_default(); for id in stale { if let Some(descriptor) = inner.descriptors.remove(&id) { - inner.ids_by_facts.remove(&descriptor_facts(descriptor)); + remove_id_from_facts_index(&mut inner, descriptor_facts(descriptor), id); } } } @@ -1041,6 +1268,118 @@ mod descriptor_tests_8067 { crate::string::js_string_from_bytes(name.as_ptr(), name.len() as u32) } + #[test] + fn every_keyless_runtime_allocator_publishes_a_shape_id() { + let _lock = crate::gc::global_side_table_test_lock(); + unsafe { + for obj in [ + crate::object::js_object_alloc(0, 0), + crate::object::js_object_alloc_fast(0, 0), + crate::object::js_object_alloc_with_parent(0x8067_0101, 0, 0), + crate::object::js_object_alloc_fast_with_parent(0x8067_0102, 0, 0), + ] { + let id = object_shape_id(obj); + assert!(is_shape_id(id), "newborn keyless object has no ShapeId"); + let facts = object_shape_descriptor(obj).expect("keyless descriptor"); + assert_eq!(facts.keys, 0); + assert_eq!(facts.logical_key_count, 0); + assert_eq!(facts.live_inline_slot_count, 0); + } + } + } + + #[test] + fn descriptor_and_prototype_changes_mint_semantic_successors() { + let _lock = crate::gc::global_side_table_test_lock(); + unsafe { + let obj = crate::object::js_object_alloc(0, 1); + crate::object::js_object_set_field_by_name(obj, key("semantic8067"), 1.0); + let structural = object_shape_id(obj); + + crate::object::descriptor_state::set_property_attrs( + obj as usize, + "semantic8067".to_string(), + crate::object::descriptor_state::PropertyAttrs::new(false, true, true), + ); + let described = object_shape_id(obj); + assert_ne!(described, structural); + let described_facts = object_shape_descriptor(obj).unwrap(); + assert_ne!(described_facts.semantic_generation, 0); + + crate::object::prototype_chain::object_set_static_prototype( + obj as usize, + crate::value::TAG_NULL, + ); + let reparented = object_shape_id(obj); + assert_ne!(reparented, described); + assert_eq!( + object_shape_descriptor(obj).unwrap().keys, + described_facts.keys, + "semantic transitions must preserve the rooted ordered keys edge" + ); + } + } + + #[test] + fn absent_descriptor_clears_do_not_mint_semantic_successors() { + let _lock = crate::gc::global_side_table_test_lock(); + unsafe { + let obj = crate::object::js_object_alloc(0, 1); + let addr = obj as usize; + let initial = object_shape_id(obj); + + crate::object::descriptor_state::clear_property_attrs(addr, "missing8067"); + crate::object::descriptor_state::clear_accessor_descriptor(addr, "missing8067"); + assert_eq!(object_shape_id(obj), initial); + + crate::object::descriptor_state::set_property_attrs( + addr, + "attrs8067".to_string(), + crate::object::descriptor_state::PropertyAttrs::new(false, true, true), + ); + crate::object::descriptor_state::clear_property_attrs(addr, "attrs8067"); + let after_real_attr_clear = object_shape_id(obj); + crate::object::descriptor_state::clear_property_attrs(addr, "attrs8067"); + assert_eq!(object_shape_id(obj), after_real_attr_clear); + + crate::object::descriptor_state::set_accessor_descriptor( + addr, + "accessor8067".to_string(), + crate::object::descriptor_state::AccessorDescriptor::default(), + ); + crate::object::descriptor_state::clear_accessor_descriptor(addr, "accessor8067"); + let after_real_accessor_clear = object_shape_id(obj); + crate::object::descriptor_state::clear_accessor_descriptor(addr, "accessor8067"); + assert_eq!(object_shape_id(obj), after_real_accessor_clear); + } + } + + #[test] + fn delete_compaction_never_compares_equal_to_the_predelete_layout() { + let _lock = crate::gc::global_side_table_test_lock(); + unsafe { + let obj = crate::object::js_object_alloc(0, 3); + let a = key("delete8067_a"); + let b = key("delete8067_b"); + let c = key("delete8067_c"); + crate::object::js_object_set_field_by_name(obj, a, 1.0); + crate::object::js_object_set_field_by_name(obj, b, 2.0); + crate::object::js_object_set_field_by_name(obj, c, 3.0); + let before = object_shape_id(obj); + assert_eq!(crate::object::js_object_delete_field(obj, a), 1); + let after = object_shape_id(obj); + assert_ne!(after, before); + let facts = object_shape_descriptor(obj).unwrap(); + assert_eq!(facts.logical_key_count, 2); + assert_eq!(facts.live_inline_slot_count, 2); + assert_eq!( + crate::object::js_object_get_field_by_name_f64(obj, b), + 2.0, + "middle-field lookup used a stale pre-delete slot mapping" + ); + } + } + #[test] fn exhaustion_parks_without_reuse_or_alias() { let next = std::sync::atomic::AtomicU32::new(SHAPE_ID_END - 1); @@ -1054,6 +1393,40 @@ mod descriptor_tests_8067 { ); } + #[test] + fn inconsistent_facts_are_not_reported_as_id_exhaustion() { + assert_eq!( + shape_descriptor_ensure(std::ptr::null(), 1, 1), + Err(ShapeDescriptorError::InvalidFacts) + ); + } + + #[test] + fn equivalent_local_and_external_ids_remain_resolvable() { + let _lock = crate::gc::global_side_table_test_lock(); + let keys = 0x8067_0000_0000_1700usize; + let local = shape_descriptor_ensure(keys as *const ArrayHeader, 1, 1) + .expect("shape range unexpectedly exhausted"); + let external = alloc_shape_id().expect("shape range unexpectedly exhausted"); + assert!(install_external_shape_id( + external, + keys as *const ArrayHeader, + 1, + 1, + )); + + assert_eq!( + shape_descriptor_ensure(keys as *const ArrayHeader, 1, 1).unwrap(), + external, + "the process-global id should be preferred for later births" + ); + retain_key_count_versions(keys as u64); + assert!(shape_descriptor_by_id(local).is_some()); + assert!(shape_descriptor_by_id(external).is_some()); + + test_drop_shape_descriptors(keys); + } + #[test] fn a_foreign_agent_id_misses_instead_of_aliasing_same_address() { let _lock = crate::gc::global_side_table_test_lock(); @@ -1079,6 +1452,31 @@ mod descriptor_tests_8067 { test_drop_shape_descriptors(fake_keys); } + #[test] + fn process_global_module_shape_id_installs_with_agent_local_keys() { + let _lock = crate::gc::global_side_table_test_lock(); + let module_keys = 0x8067_0000_0000_1800usize; + let module_id = shape_descriptor_ensure(module_keys as *const ArrayHeader, 2, 2) + .expect("shape range unexpectedly exhausted"); + let worker_keys = 0x8067_0000_0000_1900usize; + std::thread::spawn(move || { + assert!(install_external_shape_id( + module_id, + worker_keys as *const ArrayHeader, + 2, + 2, + )); + assert_eq!( + shape_descriptor_by_id(module_id).unwrap().keys, + worker_keys as u64, + "worker resolved a module ShapeId to another agent's keys pointer" + ); + }) + .join() + .expect("worker shape installation panicked"); + test_drop_shape_descriptors(module_keys); + } + #[test] fn gc_descriptor_mirror_requires_exact_release_facts() { let _lock = crate::gc::global_side_table_test_lock(); @@ -1136,7 +1534,7 @@ mod descriptor_tests_8067 { } #[test] - fn key_count_retirement_is_scoped_to_one_keys_identity() { + fn key_count_versions_remain_resolvable_until_the_keys_die() { let _lock = crate::gc::global_side_table_test_lock(); let keys = 0x8067_0000_0000_2100usize; let unrelated_keys = 0x8067_0000_0000_2200usize; @@ -1149,18 +1547,18 @@ mod descriptor_tests_8067 { let unrelated = shape_descriptor_ensure(unrelated_keys as *const ArrayHeader, 1, 1) .expect("shape range unexpectedly exhausted"); - retire_key_count_versions(keys as u64, 2); + retain_key_count_versions(keys as u64); - assert_eq!(shape_descriptor_by_id(stale_a), None); - assert_eq!(shape_descriptor_by_id(stale_b), None); + assert!(shape_descriptor_by_id(stale_a).is_some()); + assert!(shape_descriptor_by_id(stale_b).is_some()); assert!(shape_descriptor_by_id(current).is_some()); assert!(shape_descriptor_by_id(unrelated).is_some()); let inner = crate::state::state().shapes.inner.borrow(); let current_ids = inner .ids_by_keys .get(&(keys as u64)) - .expect("current keys identity disappeared from retirement index"); - assert_eq!(current_ids.as_slice(), &[current]); + .expect("keys identity disappeared from descriptor index"); + assert_eq!(current_ids.as_slice(), &[stale_a, stale_b, current]); drop(inner); test_drop_shape_descriptors(keys); diff --git a/crates/perry-runtime/src/object/spill.rs b/crates/perry-runtime/src/object/spill.rs index 91e28dc3f3..1a0e327764 100644 --- a/crates/perry-runtime/src/object/spill.rs +++ b/crates/perry-runtime/src/object/spill.rs @@ -79,24 +79,18 @@ unsafe fn spill_store_slot(spill: *mut crate::array::ArrayHeader, index: usize, crate::gc::runtime_write_barrier_slot(spill as usize, slot as usize, vbits); } -/// Only genuine shaped objects carry a meta record at the ObjectHeader -/// offset. Exotic GC_TYPE_OBJECT aliases (RegExpHeader) and every other -/// GC type (errors, maps, ...) have unrelated bytes there — the legacy -/// side table was address-keyed and safe for ANY owner, so those owners -/// keep it (in both modes) instead of deref'ing garbage. Classification -/// via the canonical header probe, mirroring `gc_object_meta_slot`. +/// Only genuine shaped objects carry a meta record at the ObjectHeader offset. +/// Every other GC type (RegExp, errors, maps, ...) has unrelated bytes there — +/// the legacy side table was address-keyed and safe for ANY owner, so those +/// owners keep it instead of dereferencing garbage. Classification uses the +/// canonical GcHeader kind, mirroring `gc_object_meta_slot`. #[inline] pub(crate) unsafe fn spill_capable_owner(obj_ptr: usize) -> bool { if obj_ptr == 0 { return false; } match crate::value::addr_class::try_read_gc_header(obj_ptr) { - Some(h) => { - h.obj_type == crate::gc::GC_TYPE_OBJECT - && !crate::regex::regex_header_has_magic( - obj_ptr as *const crate::regex::RegExpHeader, - ) - } + Some(h) => h.obj_type == crate::gc::GC_TYPE_OBJECT, None => false, } } diff --git a/crates/perry-runtime/src/promise/then_probe.rs b/crates/perry-runtime/src/promise/then_probe.rs index eec8acde6f..14a89c8a91 100644 --- a/crates/perry-runtime/src/promise/then_probe.rs +++ b/crates/perry-runtime/src/promise/then_probe.rs @@ -502,7 +502,7 @@ unsafe fn prove_no_then(value: f64) -> Outcome { return Outcome::HasDescriptors; } let obj = addr as *const ObjectHeader; - if (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR { + if !crate::object::object_is_regular(obj as *mut ObjectHeader) { return Outcome::NotPlainObject; } if !class_id_admissible((*obj).class_id) { diff --git a/crates/perry-runtime/src/proxy/put_value.rs b/crates/perry-runtime/src/proxy/put_value.rs index f6b126018a..101bda98c5 100644 --- a/crates/perry-runtime/src/proxy/put_value.rs +++ b/crates/perry-runtime/src/proxy/put_value.rs @@ -335,7 +335,7 @@ pub extern "C" fn js_put_value_set_ic_miss( let obj = obj_addr as *mut crate::ObjectHeader; let class_id = (*obj).class_id; - if (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR + if !crate::object::object_is_regular(obj) || class_id == 0 || class_id == crate::object::NATIVE_MODULE_CLASS_ID { @@ -351,7 +351,10 @@ pub extern "C" fn js_put_value_set_ic_miss( return result; } - let keys = (*obj).keys_array; + let Some(shape) = crate::object::shapes::object_shape_descriptor(obj) else { + return result; + }; + let keys = shape.keys as usize as *mut crate::array::ArrayHeader; if keys.is_null() || (keys as u64) >> 48 != 0 { return result; } @@ -359,15 +362,14 @@ pub extern "C" fn js_put_value_set_ic_miss( return result; }; if keys_gc.obj_type != crate::gc::GC_TYPE_ARRAY - || keys_gc.gc_flags & (crate::gc::GC_FLAG_FORWARDED | crate::gc::GC_FLAG_SHAPE_SHARED) - != crate::gc::GC_FLAG_SHAPE_SHARED + || keys_gc.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 { return result; } let mut own_idx = crate::object::prop_plan::read_plan_lookup(keys as usize, key as usize); if own_idx.is_none() { - let key_count = crate::array::keys_array_len_capped_to_capacity(keys); + let key_count = shape.logical_key_count as usize; if key_count > 4096 { return result; } @@ -387,18 +389,13 @@ pub extern "C" fn js_put_value_set_ic_miss( let Some(idx) = own_idx else { return result; }; - let alloc_limit = - std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32) as usize; + let alloc_limit = shape.live_inline_slot_count as usize; if idx as usize >= alloc_limit { return result; } - let parent_class_id = (*obj).parent_class_id; - let shape_token = if crate::object::shapes::is_shape_id(parent_class_id) { - crate::object::shapes::PIC_ID_TOKEN_BIT | parent_class_id as u64 - } else { - keys as u64 - }; + let shape_token = crate::object::shapes::PIC_ID_TOKEN_BIT + | crate::object::shapes::object_shape_id(obj) as u64; // Publish the token last conceptually: a zero-initialized or stale // token cannot hit this slot until it matches this receiver's current @@ -534,25 +531,19 @@ unsafe fn dyn_ic_try_store(target: f64, token: u64, slot: u32, value: f64) -> Op } let obj = obj_addr as *mut crate::ObjectHeader; let class_id = (*obj).class_id; - if (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR + if !crate::object::object_is_regular(obj) || class_id == 0 || class_id == crate::object::NATIVE_MODULE_CLASS_ID { return None; } - let current_token = { - let parent_class_id = (*obj).parent_class_id; - if crate::object::shapes::is_shape_id(parent_class_id) { - crate::object::shapes::PIC_ID_TOKEN_BIT | parent_class_id as u64 - } else { - (*obj).keys_array as u64 - } - }; + let current_token = crate::object::shapes::PIC_ID_TOKEN_BIT + | crate::object::shapes::object_shape_id(obj) as u64; if current_token != token { return None; } - let alloc_limit = std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32); - if slot >= alloc_limit { + let shape = crate::object::shapes::object_shape_descriptor(obj)?; + if slot >= shape.live_inline_slot_count { return None; } crate::object::store_object_field_slot(obj, slot as usize, value.to_bits()); @@ -617,14 +608,17 @@ pub extern "C" fn js_put_value_set_dyn_ic_miss( } let obj = obj_addr as *mut crate::ObjectHeader; let class_id = (*obj).class_id; - if (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR + if !crate::object::object_is_regular(obj) || class_id == 0 || class_id == crate::object::NATIVE_MODULE_CLASS_ID || crate::array::object_prototype_addr_matches(obj_addr) { return result; } - let keys = (*obj).keys_array; + let Some(shape) = crate::object::shapes::object_shape_descriptor(obj) else { + return result; + }; + let keys = shape.keys as usize as *mut crate::array::ArrayHeader; if keys.is_null() || (keys as u64) >> 48 != 0 { return result; } @@ -632,8 +626,7 @@ pub extern "C" fn js_put_value_set_dyn_ic_miss( return result; }; if keys_gc.obj_type != crate::gc::GC_TYPE_ARRAY - || keys_gc.gc_flags & (crate::gc::GC_FLAG_FORWARDED | crate::gc::GC_FLAG_SHAPE_SHARED) - != crate::gc::GC_FLAG_SHAPE_SHARED + || keys_gc.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 { return result; } @@ -643,7 +636,7 @@ pub extern "C" fn js_put_value_set_dyn_ic_miss( let Some(key_bytes) = crate::string::js_string_key_bytes(key_jsval, &mut key_buf) else { return result; }; - let key_count = crate::array::keys_array_len_capped_to_capacity(keys); + let key_count = shape.logical_key_count as usize; if key_count > 4096 { return result; } @@ -661,17 +654,12 @@ pub extern "C" fn js_put_value_set_dyn_ic_miss( let Some(idx) = own_idx else { return result; }; - let alloc_limit = - std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32); + let alloc_limit = shape.live_inline_slot_count; if idx >= alloc_limit { return result; } - let parent_class_id = (*obj).parent_class_id; - let shape_token = if crate::object::shapes::is_shape_id(parent_class_id) { - crate::object::shapes::PIC_ID_TOKEN_BIT | parent_class_id as u64 - } else { - keys as u64 - }; + let shape_token = crate::object::shapes::PIC_ID_TOKEN_BIT + | crate::object::shapes::object_shape_id(obj) as u64; let c = &mut *cache; let key_bits = key.to_bits() as i64; // Preserve the empty-way sentinel invariant: never prime bits 0 @@ -813,7 +801,10 @@ fn object_array_numeric_write_slots(array: f64, keys: &[f64], count: u32) -> Opt bits: u64, ) -> Option<( *mut crate::ObjectHeader, + u32, *mut crate::array::ArrayHeader, + u32, + u32, u16, )> { if (bits & !POINTER_MASK) != POINTER_TAG { @@ -828,31 +819,39 @@ fn object_array_numeric_write_slots(array: f64, keys: &[f64], count: u32) -> Opt return None; } let obj = addr as *mut crate::ObjectHeader; - if (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR - || (*obj).class_id == 0 - || (*obj).class_id == crate::object::NATIVE_MODULE_CLASS_ID - { + if (*obj).class_id == 0 || (*obj).class_id == crate::object::NATIVE_MODULE_CLASS_ID { + return None; + } + let shape = crate::object::shapes::object_shape_descriptor(obj)?; + if shape.object_kind != crate::object::shapes::ShapeObjectKind::Ordinary { return None; } - let keys = (*obj).keys_array; + let shape_id = crate::object::shapes::object_shape_stamp(obj); + let keys = shape.keys as usize as *mut crate::array::ArrayHeader; if keys.is_null() || (keys as u64) >> 48 != 0 { return None; } let keys_gc = crate::value::addr_class::try_read_gc_header(keys as usize)?; if keys_gc.obj_type != crate::gc::GC_TYPE_ARRAY - || keys_gc.gc_flags & (crate::gc::GC_FLAG_FORWARDED | crate::gc::GC_FLAG_SHAPE_SHARED) - != crate::gc::GC_FLAG_SHAPE_SHARED + || keys_gc.gc_flags & crate::gc::GC_FLAG_FORWARDED != 0 { return None; } - Some((obj, keys, gc._reserved)) + Some(( + obj, + shape_id, + keys, + shape.logical_key_count, + shape.live_inline_slot_count, + gc._reserved, + )) } unsafe fn find_slot( keys: *mut crate::array::ArrayHeader, + key_count: u32, key: *const crate::StringHeader, ) -> Option { - let key_count = crate::array::keys_array_len_capped_to_capacity(keys); if key_count > 4096 { return None; } @@ -873,16 +872,17 @@ fn object_array_numeric_write_slots(array: f64, keys: &[f64], count: u32) -> Opt trace_object_array_numeric_write_rejection("first receiver is a hole"); return None; } - let (first, shared_keys, first_flags) = trace_object_array_numeric_write_stage( - unsafe { validated_object(first_bits) }, - "first receiver is not an eligible regular shared-shape object", - )?; + let (first, shared_shape_id, shared_keys, shared_key_count, first_limit, first_flags) = + trace_object_array_numeric_write_stage( + unsafe { validated_object(first_bits) }, + "first receiver is not an eligible regular shared-shape object", + )?; let mut slots = [0u16; 4]; for index in 0..keys.len() { // `find_slot` caps the shared keys array at 4096 entries, so every // non-zero-encoded index fits comfortably in one 16-bit result lane. let slot = trace_object_array_numeric_write_stage( - unsafe { find_slot(shared_keys, decoded_keys[index]) }, + unsafe { find_slot(shared_keys, shared_key_count, decoded_keys[index]) }, "target key is absent from the shared shape", )?; slots[index] = trace_object_array_numeric_write_stage( @@ -924,12 +924,6 @@ fn object_array_numeric_write_slots(array: f64, keys: &[f64], count: u32) -> Opt slot < (*spill).length && slot < (*spill).capacity } - let first_limit = unsafe { - std::cmp::max( - (*first).field_count, - crate::object::INLINE_SLOT_FLOOR as u32, - ) - }; let mut lane_spill = [false; 4]; for index in 0..keys.len() { let slot = u32::from(slots[index]); @@ -972,18 +966,17 @@ fn object_array_numeric_write_slots(array: f64, keys: &[f64], count: u32) -> Opt trace_object_array_numeric_write_rejection("receiver prefix contains a hole"); return None; } - let (obj, object_keys, flags) = trace_object_array_numeric_write_stage( - unsafe { validated_object(bits) }, - "receiver prefix contains an ineligible object", - )?; - if object_keys != shared_keys { + let (obj, receiver_shape_id, _object_keys, _object_key_count, limit, flags) = + trace_object_array_numeric_write_stage( + unsafe { validated_object(bits) }, + "receiver prefix contains an ineligible object", + )?; + if receiver_shape_id != shared_shape_id { trace_object_array_numeric_write_rejection( - "receiver prefix does not share one keys array", + "receiver prefix does not share one ShapeId", ); return None; } - let limit = - unsafe { std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32) }; for index in 0..keys.len() { let slot = u32::from(slots[index]); if lane_spill[index] { diff --git a/crates/perry-runtime/src/regex.rs b/crates/perry-runtime/src/regex.rs index b400acdd92..9b3a9c04ee 100644 --- a/crates/perry-runtime/src/regex.rs +++ b/crates/perry-runtime/src/regex.rs @@ -100,14 +100,11 @@ crate::perry_thread_local! { static LAST_EXEC_GROUPS: RefCell<*mut ObjectHeader> = const { RefCell::new(ptr::null_mut()) }; - /// Set of all RegExpHeader pointers ever allocated in this thread. + /// Set of live RegExpHeader pointers allocated in this thread. /// Used by callers (e.g. `js_string_split`) to distinguish a regex /// delimiter from a string delimiter when the codegen can't tell - /// statically. Pointers are never removed; RegExpHeader is backed by - /// `gc_malloc` but headers are effectively permanent in practice, and - /// even if a header is freed, subsequent lookups will simply miss — - /// the worst outcome is that a stale regex is treated as a string - /// (safe) rather than the other way around (segfault). + /// statically. GC move/death hooks rekey and remove entries as cells + /// relocate or die. Header magic remains the primary identity check. static REGEX_POINTERS: RefCell> = RefCell::new(HashSet::new()); /// Issue #637: Owned copies of pattern and flags strings keyed by @@ -131,7 +128,7 @@ pub(crate) fn is_regex_pointer(ptr: *const u8) -> bool { } // Wall 18: check the header-resident magic FIRST so identity survives a // duplicate-runtime thread-local split (see `RegExpHeader.magic`). A - // RegExp is a `gc_malloc(GC_TYPE_OBJECT)` allocation, so it always carries + // RegExp is a GC-tracked `GC_TYPE_REGEXP` allocation, so it always carries // a preceding GcHeader; only read the magic field when the GC header says // this is an object of sufficient size to actually contain it. if regex_header_has_magic(ptr as *const RegExpHeader) { @@ -159,8 +156,91 @@ fn regex_pointers_contains(addr: usize) -> bool { REGEX_POINTERS.with(|s| s.borrow().contains(&addr)) } +/// Rekey every address-owned RegExp table after payload evacuation. Header +/// child slots are rewritten separately by the RegExp GC descriptor; this +/// hook handles the owner keys that a slot visitor cannot see. +pub(crate) fn regex_header_moved_for_gc(old_addr: usize, new_addr: usize) { + if old_addr == new_addr { + return; + } + REGEX_POINTERS.with(|table| { + let mut table = table.borrow_mut(); + if table.remove(&old_addr) { + table.insert(new_addr); + } + }); + REGEX_SOURCE_TABLE.with(|table| { + let mut table = table.borrow_mut(); + if let Some(source) = table.remove(&old_addr) { + table.insert(new_addr, source); + } + }); + crate::object::exotic_expando::exotic_expando_owner_moved(old_addr, new_addr); +} + +/// Remove address-owned RegExp metadata when the cell is proven dead. +pub(crate) fn regex_header_clear_dead_for_gc(addr: usize) { + REGEX_POINTERS.with(|table| { + table.borrow_mut().remove(&addr); + }); + REGEX_SOURCE_TABLE.with(|table| { + table.borrow_mut().remove(&addr); + }); + crate::object::exotic_expando::exotic_expando_owner_clear_dead(addr); +} + +#[cfg(test)] +pub(crate) fn test_regex_pointer_entry_exists(addr: usize) -> bool { + REGEX_POINTERS.with(|table| table.borrow().contains(&addr)) +} + +#[cfg(test)] +pub(crate) fn test_regex_source_entry_exists(addr: usize) -> bool { + REGEX_SOURCE_TABLE.with(|table| table.borrow().contains_key(&addr)) +} + +/// Build a minimal nursery-resident RegExp payload for the copying collector's +/// relocation contract test. Production construction currently chooses the +/// malloc-backed arm of `ArenaOrMalloc`; this exercises the same registered GC +/// type through its arena arm so future allocator routing cannot silently +/// strand the address-owned tables. +#[cfg(all(test, feature = "regex-engine"))] +pub(crate) fn test_alloc_nursery_regexp_for_move(source: &str, flags: &str) -> *mut RegExpHeader { + unsafe { + let ptr = crate::arena::arena_alloc_gc( + std::mem::size_of::(), + std::mem::align_of::(), + crate::gc::GC_TYPE_REGEXP, + ) as *mut RegExpHeader; + (*ptr).regex_ptr = std::ptr::null_mut(); + (*ptr).pattern_ptr = std::ptr::null(); + (*ptr).flags_ptr = std::ptr::null(); + (*ptr).case_insensitive = flags.contains('i'); + (*ptr).global = flags.contains('g'); + (*ptr).multiline = flags.contains('m'); + (*ptr).sticky = flags.contains('y'); + (*ptr).dot_all = flags.contains('s'); + (*ptr).unicode = flags.contains('u') || flags.contains('v'); + (*ptr).has_indices = flags.contains('d'); + (*ptr).last_index = crate::value::JSValue::number(0.0).bits(); + (*ptr).magic = REGEXP_MAGIC; + (*ptr).fancy_ptr = std::ptr::null(); + + REGEX_EVER_REGISTERED.arm(); + REGEX_POINTERS.with(|table| { + table.borrow_mut().insert(ptr as usize); + }); + REGEX_SOURCE_TABLE.with(|table| { + table + .borrow_mut() + .insert(ptr as usize, (source.to_string(), flags.to_string())); + }); + ptr + } +} + /// Bounds-checked read of `RegExpHeader.magic`. Confirms the preceding -/// `GcHeader` exists, is a `GC_TYPE_OBJECT`, and the allocation is large enough +/// `GcHeader` exists, is a `GC_TYPE_REGEXP`, and the allocation is large enough /// to hold a full `RegExpHeader` before dereferencing the `magic` field. /// Returns true iff the field equals [`REGEXP_MAGIC`]. Immune to which linked /// `perry-runtime` copy's thread-locals are live. @@ -180,7 +260,7 @@ pub(crate) fn regex_header_has_magic(re: *const RegExpHeader) -> bool { let Some(gc) = crate::value::addr_class::try_read_gc_header(addr) else { return false; }; - if gc.obj_type != crate::gc::GC_TYPE_OBJECT { + if gc.obj_type != crate::gc::GC_TYPE_REGEXP { return false; } // `size` in the GcHeader covers the GcHeader + payload. Require enough @@ -522,7 +602,7 @@ pub(crate) fn is_valid_regex_ptr(p: *const RegExpHeader) -> bool { /// Public: is `addr` a RegExpHeader we allocated via `js_regexp_new`? /// Used by the console/`util.inspect` formatter to print regex literals -/// as `/source/flags` instead of `{}` (they're GC_TYPE_OBJECT allocations +/// as `/source/flags` instead of `{}` (they're GC_TYPE_REGEXP allocations /// with no enumerable string keys). Registry-gated so a generic object /// is never mis-read as a RegExpHeader. pub fn is_registered_regex(addr: usize) -> bool { @@ -806,7 +886,7 @@ pub extern "C" fn js_regexp_new( // missing is that the value written had to survive the allocation first. let flags_root = scope.root_string_ptr(canonical_flags_ptr); unsafe { - let raw = crate::gc::gc_malloc(header_size, crate::gc::GC_TYPE_OBJECT); + let raw = crate::gc::gc_malloc(header_size, crate::gc::GC_TYPE_REGEXP); if raw.is_null() { // #5067 — catchable RangeError instead of aborting on OOM. crate::error::throw_allocation_failed(); diff --git a/crates/perry-runtime/src/regex/tests.rs b/crates/perry-runtime/src/regex/tests.rs index 201914bd6b..c69e8d42fa 100644 --- a/crates/perry-runtime/src/regex/tests.rs +++ b/crates/perry-runtime/src/regex/tests.rs @@ -5,6 +5,50 @@ fn make_string(s: &str) -> *mut StringHeader { js_string_from_bytes(s.as_ptr(), s.len() as u32) } +#[test] +fn regexp_has_dedicated_gc_kind_and_is_not_a_shaped_object() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let pattern = scope.root_string_ptr(make_string("x")); + let flags = scope.root_string_ptr(make_string("g")); + let re = pattern.with_mut_ptr::(|pattern| { + flags.with_mut_ptr::(|flags| js_regexp_new(pattern, flags)) + }); + let gc = unsafe { crate::value::addr_class::try_read_gc_header(re as usize) } + .expect("RegExp must be a GC allocation"); + assert_eq!(gc.obj_type, crate::gc::GC_TYPE_REGEXP); + assert!(regex_header_has_magic(re)); + assert!(!unsafe { crate::object::object_is_shaped(re.cast::()) }); +} + +#[test] +fn malloc_finalize_clears_regexp_address_owned_tables() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let pattern = scope.root_string_ptr(make_string("finalize")); + let flags = scope.root_string_ptr(make_string("g")); + let re = pattern.with_mut_ptr::(|pattern| { + flags.with_mut_ptr::(|flags| js_regexp_new(pattern, flags)) + }); + let addr = re as usize; + assert!(test_regex_pointer_entry_exists(addr)); + assert!(test_regex_source_entry_exists(addr)); + crate::object::exotic_expando::test_seed_exotic_expando_entry( + addr, + "owned", + crate::value::TAG_TRUE, + ); + assert!(crate::object::exotic_expando::test_exotic_expando_entry_exists(addr)); + + unsafe { + crate::gc::gc_type_finalize_unmarked_payload(crate::gc::GC_TYPE_REGEXP, re.cast::()); + } + + assert!(!test_regex_pointer_entry_exists(addr)); + assert!(!test_regex_source_entry_exists(addr)); + assert!(!crate::object::exotic_expando::test_exotic_expando_entry_exists(addr)); +} + #[test] fn js_replacement_expands_special_patterns() { let re = regex::Regex::new(r"(\w+)\s(\w+)").unwrap(); diff --git a/crates/perry-runtime/src/typed_feedback.rs b/crates/perry-runtime/src/typed_feedback.rs index 3318191c61..eebff17426 100644 --- a/crates/perry-runtime/src/typed_feedback.rs +++ b/crates/perry-runtime/src/typed_feedback.rs @@ -765,70 +765,17 @@ fn object_shape(addr: usize) -> (usize, u32, u16) { return (0, 0, gc_type); } let class_id = (*ptr).class_id; - // #6804: plain objects canonicalize the token on the stable - // ShapeId. Shape-cached literals are stamped at birth; anything - // else is stamped HERE on first observation (self-healing), so one - // logical shape can never split into a pre-stamp address token and - // a post-stamp id token within a site. - // - // ★ #6759 C3 rung 1 deliberately does NOT relax this `class_id == 0` - // gate, even though rung 1 gives class instances a shape word. This - // token is not a PIC token — it is compared against a CODEGEN-SUPPLIED - // KEYS POINTER (`@perry_class_keys_C`) by the typed_feedback guard - // family: `guards.rs::method_direct_call_contract` requires - // `shape_addr == expected_keys as usize`, and the class-field / - // element-shape contracts do the same. An id can never equal that - // pointer, so returning one here fails every such guard CLOSED — - // memory-safe, but it silently deletes the direct-method-call route - // and the class-field fast paths. (Both are pinned: - // `typed_feedback_method_direct_guard_passes_for_exact_registered_method` - // and `typed_feedback_class_field_get_guard_requires_raw_f64_layout_when_requested` - // go red the moment this gate is dropped.) - // - // Switching those consumers from a keys pointer to a ShapeId is - // rung 3 — nine unvalidated consumers, its own review. The PIC-token - // half of the observable rung 1 wanted comes from `ic_miss.rs`, which - // primes what the emitted PIC actually computes; this function feeds - // observation and guards, which are a different population. - let shape = if class_id == 0 { - let stamp = crate::object::shapes::object_shape_stamp(ptr); - if stamp != 0 { - stamp as usize - } else if crate::regex::regex_header_has_magic( - addr as *const crate::regex::RegExpHeader, - ) { - // RegExpHeader aliases GC_TYPE_OBJECT with a different - // layout — never write through the ObjectHeader view; keep - // the legacy (equality-only) address token. - (*ptr).keys_array as usize - } else { - let keys = (*ptr).keys_array; - if let Some(keys_header) = - crate::value::addr_class::try_read_gc_header(keys as usize) - { - if keys_header.obj_type == crate::gc::GC_TYPE_ARRAY - || keys_header.obj_type == crate::gc::GC_TYPE_LAZY_ARRAY - { - let id = crate::object::shapes::stamp_object_shape( - ptr as *mut ObjectHeader, - keys, - (*keys).length, - ); - if id != 0 { - id as usize - } else { - keys as usize - } - } else { - keys as usize - } - } else { - keys as usize - } - } - } else { - (*ptr).keys_array as usize - }; + // #8067 rung 3: every genuine ObjectHeader uses one token domain. + // Runtime allocators birth-stamp objects; the synchronization call is + // a defensive self-heal for old/synthetic callers and never falls back + // to a keys pointer. + let mut shape = crate::object::shapes::object_shape_id(ptr); + if shape == 0 { + shape = crate::object::shapes::synchronize_object_shape_descriptor( + ptr as *mut ObjectHeader, + ); + } + let shape = shape as usize; (shape, class_id, gc_type) } } @@ -1720,15 +1667,13 @@ fn object_key_matches_field( } unsafe { let obj = object_addr as *mut ObjectHeader; - let alloc_limit = - std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32); - if field_index >= alloc_limit { + let Some(descriptor) = crate::object::shapes::object_shape_descriptor(obj) else { + return false; + }; + if field_index >= descriptor.live_inline_slot_count { return false; } - let keys = (*obj).keys_array; - // #6804: `shape_addr` is an opaque TOKEN (a stable ShapeId for - // stamped plain objects), not necessarily the keys address — the - // actual contract is carried by the key/slot validation below. + let keys = descriptor.keys as usize as *const ArrayHeader; if keys.is_null() { return false; } diff --git a/crates/perry-runtime/src/typed_feedback/guards.rs b/crates/perry-runtime/src/typed_feedback/guards.rs index cb2476caa3..24a52ae4c9 100644 --- a/crates/perry-runtime/src/typed_feedback/guards.rs +++ b/crates/perry-runtime/src/typed_feedback/guards.rs @@ -11,9 +11,10 @@ fn object_has_own_key_bytes(obj: *const ObjectHeader, key_bytes: &[u8]) -> bool } unsafe { let obj = object_addr as *const ObjectHeader; - let keys = (*obj).keys_array; - // #6804: `shape_addr` is an opaque token (see `object_shape`), not - // necessarily the keys address — the key scan below is the check. + let Some(descriptor) = crate::object::shapes::object_shape_descriptor(obj) else { + return false; + }; + let keys = descriptor.keys as usize as *const ArrayHeader; if keys.is_null() { return false; } @@ -85,7 +86,7 @@ fn prototype_may_override_method(class_id: u32, method_name: &str, method_bytes: fn method_direct_call_contract( receiver: f64, expected_class_id: u32, - expected_keys: *const ArrayHeader, + expected_shape_id: u32, method_name_ptr: *const i8, method_name_len: usize, expected_func_ptr: *const u8, @@ -107,7 +108,7 @@ fn method_direct_call_contract( let name_hash = hash_bytes(method_bytes); if object_addr == 0 || expected_class_id == 0 - || expected_keys.is_null() + || !crate::object::shapes::is_shape_id(expected_shape_id) || expected_func_ptr.is_null() { return (shape_addr, class_id, gc_type, name_hash, false); @@ -122,13 +123,13 @@ fn method_direct_call_contract( return (shape_addr, class_id, gc_type, name_hash, false); } let obj = object_addr as *const ObjectHeader; - if (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR { + if !crate::object::object_is_regular(obj) { return (shape_addr, class_id, gc_type, name_hash, false); } if (*obj).class_id == crate::object::NATIVE_MODULE_CLASS_ID || (*obj).class_id != expected_class_id - || !std::ptr::eq((*obj).keys_array, expected_keys) - || shape_addr != expected_keys as usize + || crate::object::shapes::object_shape_id(obj) != expected_shape_id + || shape_addr != expected_shape_id as usize { return (shape_addr, class_id, gc_type, name_hash, false); } @@ -261,13 +262,16 @@ fn class_field_raw_f64_layout_contract( fn class_field_get_contract( receiver: f64, expected_class_id: u32, - expected_keys: *const ArrayHeader, + expected_shape_id: u32, key: *const crate::StringHeader, expected_field_index: u32, require_raw_f64: bool, ) -> (usize, u32, u16, bool) { let object_addr = normalize_raw_object_addr(receiver.to_bits()); - if object_addr == 0 || expected_class_id == 0 || expected_keys.is_null() { + if object_addr == 0 + || expected_class_id == 0 + || !crate::object::shapes::is_shape_id(expected_shape_id) + { return (0, 0, 0, false); } let Some(gc_header) = gc_header_for_user_addr(object_addr) else { @@ -284,17 +288,21 @@ fn class_field_get_contract( let obj = object_addr as *mut ObjectHeader; let class_id = (*obj).class_id; - let shape_addr = (*obj).keys_array as usize; + let shape_id = crate::object::shapes::object_shape_id(obj); + let shape_addr = shape_id as usize; + let Some(descriptor) = crate::object::shapes::shape_descriptor_by_id(shape_id) else { + return (shape_addr, class_id, gc_type, false); + }; let key_name = match key_as_str(key) { Some(name) => name, None => return (shape_addr, class_id, gc_type, false), }; - let expected_shape_addr = expected_keys as usize; - let valid = (*obj).object_type == crate::error::OBJECT_TYPE_REGULAR + let keys = descriptor.keys as usize as *const ArrayHeader; + let valid = crate::object::object_is_regular(obj) && class_id == expected_class_id - && shape_addr == expected_shape_addr - && expected_field_index < (*obj).field_count - && plain_array_index_guard(expected_keys, expected_field_index, true) + && shape_id == expected_shape_id + && expected_field_index < descriptor.live_inline_slot_count + && plain_array_index_guard(keys, expected_field_index, true) && object_key_matches_field(obj, key, expected_field_index) && class_field_raw_f64_layout_contract( object_addr, @@ -310,12 +318,15 @@ fn class_field_get_contract( fn class_field_fast_contract( receiver: f64, expected_class_id: u32, - expected_keys: *const ArrayHeader, + expected_shape_id: u32, expected_field_index: u32, require_raw_f64: bool, ) -> bool { let object_addr = normalize_raw_object_addr(receiver.to_bits()); - if object_addr == 0 || expected_class_id == 0 || expected_keys.is_null() { + if object_addr == 0 + || expected_class_id == 0 + || !crate::object::shapes::is_shape_id(expected_shape_id) + { return false; } let Some(gc_header) = gc_header_for_user_addr(object_addr) else { @@ -328,10 +339,14 @@ fn class_field_fast_contract( return false; } let obj = object_addr as *const ObjectHeader; - let shape_ok = (*obj).object_type == crate::error::OBJECT_TYPE_REGULAR - && (*obj).class_id == expected_class_id - && std::ptr::eq((*obj).keys_array as *const ArrayHeader, expected_keys) - && expected_field_index < (*obj).field_count; + let descriptor = crate::object::shapes::object_shape_descriptor(obj); + let shape_id = crate::object::shapes::object_shape_stamp(obj); + let shape_ok = (*obj).class_id == expected_class_id + && shape_id == expected_shape_id + && descriptor.is_some_and(|facts| { + facts.object_kind == crate::object::shapes::ShapeObjectKind::Ordinary + && expected_field_index < facts.live_inline_slot_count + }); let layout_ok = shape_ok && class_field_raw_f64_layout_contract( object_addr, @@ -394,7 +409,7 @@ pub extern "C" fn js_typed_feedback_class_field_get_guard( site_id: u64, receiver: f64, expected_class_id: u32, - expected_keys: *const ArrayHeader, + expected_shape_id: u32, key: *const crate::StringHeader, expected_field_index: u32, require_raw_f64: i32, @@ -403,7 +418,7 @@ pub extern "C" fn js_typed_feedback_class_field_get_guard( return class_field_fast_contract( receiver, expected_class_id, - expected_keys, + expected_shape_id, expected_field_index, require_raw_f64 != 0, ) as i32; @@ -411,7 +426,7 @@ pub extern "C" fn js_typed_feedback_class_field_get_guard( let (shape_addr, class_id, gc_type, contract_valid) = class_field_get_contract( receiver, expected_class_id, - expected_keys, + expected_shape_id, key, expected_field_index, require_raw_f64 != 0, @@ -442,7 +457,7 @@ pub extern "C" fn js_typed_feedback_class_field_get_guard( fn class_field_set_fast_contract( receiver: f64, expected_class_id: u32, - expected_keys: *const ArrayHeader, + expected_shape_id: u32, expected_field_index: u32, require_raw_f64: bool, value_bits: u64, @@ -451,7 +466,7 @@ fn class_field_set_fast_contract( if !class_field_fast_contract( receiver, expected_class_id, - expected_keys, + expected_shape_id, expected_field_index, require_raw_f64, ) { @@ -508,14 +523,17 @@ fn descriptor_blocks_class_field_set(obj_addr: usize, class_id: u32, key_name: & fn class_field_set_contract( receiver: f64, expected_class_id: u32, - expected_keys: *const ArrayHeader, + expected_shape_id: u32, key: *const crate::StringHeader, expected_field_index: u32, require_raw_f64: bool, value_bits: u64, ) -> (usize, u32, u16, bool) { let object_addr = normalize_raw_object_addr(receiver.to_bits()); - if object_addr == 0 || expected_class_id == 0 || expected_keys.is_null() { + if object_addr == 0 + || expected_class_id == 0 + || !crate::object::shapes::is_shape_id(expected_shape_id) + { return (0, 0, 0, false); } let Some(gc_header) = gc_header_for_user_addr(object_addr) else { @@ -531,21 +549,31 @@ fn class_field_set_contract( } if (*gc_header)._reserved & crate::gc::OBJ_FLAG_FROZEN != 0 { let obj = object_addr as *mut ObjectHeader; - return ((*obj).keys_array as usize, (*obj).class_id, gc_type, false); + return ( + crate::object::shapes::object_shape_id(obj) as usize, + (*obj).class_id, + gc_type, + false, + ); } let obj = object_addr as *mut ObjectHeader; let class_id = (*obj).class_id; - let shape_addr = (*obj).keys_array as usize; + let shape_id = crate::object::shapes::object_shape_id(obj); + let shape_addr = shape_id as usize; + let Some(descriptor) = crate::object::shapes::shape_descriptor_by_id(shape_id) else { + return (shape_addr, class_id, gc_type, false); + }; let key_name = match key_as_str(key) { Some(name) => name, None => return (shape_addr, class_id, gc_type, false), }; - let expected_shape_addr = expected_keys as usize; + let keys = descriptor.keys as usize as *const ArrayHeader; let valid = class_id == expected_class_id - && shape_addr == expected_shape_addr - && expected_field_index < (*obj).field_count - && plain_array_index_guard(expected_keys, expected_field_index, true) + && crate::object::object_is_regular(obj) + && shape_id == expected_shape_id + && expected_field_index < descriptor.live_inline_slot_count + && plain_array_index_guard(keys, expected_field_index, true) && object_key_matches_field(obj, key, expected_field_index) && (!require_raw_f64 || (is_plain_number_bits(value_bits) @@ -565,7 +593,7 @@ pub extern "C" fn js_typed_feedback_class_field_set_guard( site_id: u64, receiver: f64, expected_class_id: u32, - expected_keys: *const ArrayHeader, + expected_shape_id: u32, key: *const crate::StringHeader, expected_field_index: u32, value: f64, @@ -576,7 +604,7 @@ pub extern "C" fn js_typed_feedback_class_field_set_guard( return class_field_set_fast_contract( receiver, expected_class_id, - expected_keys, + expected_shape_id, expected_field_index, require_raw_f64 != 0, value_bits, @@ -585,7 +613,7 @@ pub extern "C" fn js_typed_feedback_class_field_set_guard( let (shape_addr, class_id, gc_type, contract_valid) = class_field_set_contract( receiver, expected_class_id, - expected_keys, + expected_shape_id, key, expected_field_index, require_raw_f64 != 0, @@ -687,7 +715,7 @@ pub extern "C" fn js_class_field_set_ic( site_id: u64, receiver: f64, expected_class_id: u32, - expected_keys: *const ArrayHeader, + expected_shape_id: u32, key: *const crate::StringHeader, expected_field_index: u32, value: f64, @@ -697,7 +725,7 @@ pub extern "C" fn js_class_field_set_ic( site_id, receiver, expected_class_id, - expected_keys, + expected_shape_id, key, expected_field_index, value, @@ -757,7 +785,7 @@ pub extern "C" fn js_class_field_get_ic( site_id: u64, receiver: f64, expected_class_id: u32, - expected_keys: *const ArrayHeader, + expected_shape_id: u32, key: *const crate::StringHeader, expected_field_index: u32, require_raw_f64: i32, @@ -766,7 +794,7 @@ pub extern "C" fn js_class_field_get_ic( site_id, receiver, expected_class_id, - expected_keys, + expected_shape_id, key, expected_field_index, require_raw_f64, @@ -949,7 +977,7 @@ pub unsafe extern "C" fn js_typed_feedback_method_direct_call_guard( site_id: u64, receiver: f64, expected_class_id: u32, - expected_keys: *const ArrayHeader, + expected_shape_id: u32, method_name_ptr: *const i8, method_name_len: usize, expected_func_ptr: *const u8, @@ -958,7 +986,7 @@ pub unsafe extern "C" fn js_typed_feedback_method_direct_call_guard( let (shape_addr, class_id, gc_type, name_hash, contract_valid) = method_direct_call_contract( receiver, expected_class_id, - expected_keys, + expected_shape_id, method_name_ptr, method_name_len, expected_func_ptr, @@ -987,14 +1015,14 @@ pub unsafe extern "C" fn js_typed_feedback_method_direct_call_guard( } /// The class-id half of [`js_method_direct_shape_guard`], hoisted out so a -/// call site can test MORE than one (class id, keys token) pair per probe. +/// call site can test MORE than one (class id, ShapeId) pair per probe. /// /// Returns the receiver's `class_id` when every precondition the guard checks -/// *other than* the class-id / keys comparison holds, and writes the -/// receiver's `keys_array` pointer through `out_keys`. Returns 0 — never a +/// *other than* the class-id / shape comparison holds, and writes the +/// receiver's ShapeId through `out_shape_id`. Returns 0 — never a /// valid user class id — when any precondition fails, and then leaves -/// `*out_keys` at 0 so a caller that skips the return check still cannot match -/// a real keys token. +/// output at 0 so a caller that skips the return check still cannot match a +/// real ShapeId. /// /// This exists because the single-pair guard speculates the receiver's dynamic /// class is exactly the *declared* class of the expression. For a receiver @@ -1005,9 +1033,12 @@ pub unsafe extern "C" fn js_typed_feedback_method_direct_call_guard( /// the same information into a direct call. See /// `perry-codegen/src/lower_call/method_override.rs`. #[no_mangle] -pub unsafe extern "C" fn js_method_direct_shape_class(receiver: f64, out_keys: *mut u64) -> u32 { - if !out_keys.is_null() { - *out_keys = 0; +pub unsafe extern "C" fn js_method_direct_shape_class( + receiver: f64, + out_shape_id: *mut u32, +) -> u32 { + if !out_shape_id.is_null() { + *out_shape_id = 0; } let object_addr = normalize_raw_object_addr(receiver.to_bits()); if object_addr == 0 { @@ -1024,15 +1055,19 @@ pub unsafe extern "C" fn js_method_direct_shape_class(receiver: f64, out_keys: * return 0; } let obj = object_addr as *const ObjectHeader; - if (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR { + if !crate::object::object_is_regular(obj) { return 0; } let class_id = (*obj).class_id; if class_id == 0 { return 0; } - if !out_keys.is_null() { - *out_keys = (*obj).keys_array as u64; + let shape_id = crate::object::shapes::object_shape_id(obj); + if shape_id == 0 { + return 0; + } + if !out_shape_id.is_null() { + *out_shape_id = shape_id; } class_id } @@ -1041,14 +1076,14 @@ pub unsafe extern "C" fn js_method_direct_shape_class(receiver: f64, out_keys: * pub unsafe extern "C" fn js_method_direct_shape_guard( receiver: f64, expected_class_id: u32, - expected_keys: *const ArrayHeader, + expected_shape_id: u32, ) -> i32 { - if expected_class_id == 0 || expected_keys.is_null() { + if expected_class_id == 0 || !crate::object::shapes::is_shape_id(expected_shape_id) { return 0; } - let mut keys: u64 = 0; - let class_id = js_method_direct_shape_class(receiver, &mut keys); - (class_id == expected_class_id && keys == expected_keys as u64) as i32 + let mut shape_id = 0; + let class_id = js_method_direct_shape_class(receiver, &mut shape_id); + (class_id == expected_class_id && shape_id == expected_shape_id) as i32 } #[no_mangle] @@ -1124,21 +1159,21 @@ pub extern "C" fn js_typed_feedback_closure_direct_call_guard( mod keep_guard_symbols { use super::*; #[cfg(feature = "keepalive-anchors")] -#[used] static G0: extern "C" fn(u64, f64, u32, *const ArrayHeader, *const crate::StringHeader, u32, i32) -> i32 = js_typed_feedback_class_field_get_guard; + #[used] static G0: extern "C" fn(u64, f64, u32, u32, *const crate::StringHeader, u32, i32) -> i32 = js_typed_feedback_class_field_get_guard; #[cfg(feature = "keepalive-anchors")] -#[used] static G1: extern "C" fn(u64, f64, u32, *const ArrayHeader, *const crate::StringHeader, u32, f64, i32) -> i32 = js_typed_feedback_class_field_set_guard; + #[used] static G1: extern "C" fn(u64, f64, u32, u32, *const crate::StringHeader, u32, f64, i32) -> i32 = js_typed_feedback_class_field_set_guard; #[cfg(feature = "keepalive-anchors")] -#[used] static G1C: extern "C" fn(u64, u64, u64, f64) = js_class_field_set_fallback; + #[used] static G1C: extern "C" fn(u64, u64, u64, f64) = js_class_field_set_fallback; #[cfg(feature = "keepalive-anchors")] -#[used] static G1D: extern "C" fn(u64, f64, u32, *const ArrayHeader, *const crate::StringHeader, u32, f64, i32) = js_class_field_set_ic; + #[used] static G1D: extern "C" fn(u64, f64, u32, u32, *const crate::StringHeader, u32, f64, i32) = js_class_field_set_ic; #[cfg(feature = "keepalive-anchors")] -#[used] static G1E: extern "C" fn(u64, f64, u32, *const ArrayHeader, *const crate::StringHeader, u32, i32) -> f64 = js_class_field_get_ic; + #[used] static G1E: extern "C" fn(u64, f64, u32, u32, *const crate::StringHeader, u32, i32) -> f64 = js_class_field_get_ic; #[cfg(feature = "keepalive-anchors")] -#[used] static G2: unsafe extern "C" fn(u64, f64, u32, *const ArrayHeader, *const i8, usize, *const u8) -> i32 = js_typed_feedback_method_direct_call_guard; + #[used] static G2: unsafe extern "C" fn(u64, f64, u32, u32, *const i8, usize, *const u8) -> i32 = js_typed_feedback_method_direct_call_guard; #[cfg(feature = "keepalive-anchors")] -#[used] static G3: extern "C" fn(u64, f64, *const u8, u32, u32) -> i32 = js_typed_feedback_closure_direct_call_guard; + #[used] static G3: extern "C" fn(u64, f64, *const u8, u32, u32) -> i32 = js_typed_feedback_closure_direct_call_guard; #[cfg(feature = "keepalive-anchors")] -#[used] static G4: unsafe extern "C" fn(f64, u32, *const ArrayHeader) -> i32 = js_method_direct_shape_guard; + #[used] static G4: unsafe extern "C" fn(f64, u32, u32) -> i32 = js_method_direct_shape_guard; #[cfg(feature = "keepalive-anchors")] -#[used] static G4B: unsafe extern "C" fn(f64, *mut u64) -> u32 = js_method_direct_shape_class; + #[used] static G4B: unsafe extern "C" fn(f64, *mut u32) -> u32 = js_method_direct_shape_class; } diff --git a/crates/perry-runtime/src/typed_feedback/tests.rs b/crates/perry-runtime/src/typed_feedback/tests.rs index 4db726e112..6941ce4f40 100644 --- a/crates/perry-runtime/src/typed_feedback/tests.rs +++ b/crates/perry-runtime/src/typed_feedback/tests.rs @@ -75,6 +75,10 @@ fn class_instance( (obj, keys, key, receiver) } +fn shape_id(obj: *const crate::object::ObjectHeader) -> u32 { + unsafe { crate::object::shapes::object_shape_id(obj) } +} + unsafe fn register_test_method(class_id: u32, name: &'static [u8]) { crate::object::js_register_class_method( class_id as i64, @@ -1249,19 +1253,19 @@ fn representation_lowering_helpers_have_lto_keepalive_anchors() { ( guards, "static G0", - "static G0: extern \"C\" fn(u64, f64, u32, *const ArrayHeader, *const crate::StringHeader, u32, i32) -> i32", + "static G0: extern \"C\" fn(u64, f64, u32, u32, *const crate::StringHeader, u32, i32) -> i32", "js_typed_feedback_class_field_get_guard", ), ( guards, "static G1", - "static G1: extern \"C\" fn(u64, f64, u32, *const ArrayHeader, *const crate::StringHeader, u32, f64, i32) -> i32", + "static G1: extern \"C\" fn(u64, f64, u32, u32, *const crate::StringHeader, u32, f64, i32) -> i32", "js_typed_feedback_class_field_set_guard", ), ( guards, "static G2", - "static G2: unsafe extern \"C\" fn(u64, f64, u32, *const ArrayHeader, *const i8, usize, *const u8) -> i32", + "static G2: unsafe extern \"C\" fn(u64, f64, u32, u32, *const i8, usize, *const u8) -> i32", "js_typed_feedback_method_direct_call_guard", ), ( @@ -1273,7 +1277,7 @@ fn representation_lowering_helpers_have_lto_keepalive_anchors() { ( guards, "static G4", - "static G4: unsafe extern \"C\" fn(f64, u32, *const ArrayHeader) -> i32", + "static G4: unsafe extern \"C\" fn(f64, u32, u32) -> i32", "js_method_direct_shape_guard", ), ( @@ -1552,12 +1556,21 @@ fn typed_feedback_class_field_set_guard_fails_for_frozen_object() { register(31, TypedFeedbackSiteKind::PropertySet, "obj.x="); let class_id = 0x7EED_0031; - let (obj, keys, key, receiver) = class_instance(class_id, b"x"); + let (obj, _, key, receiver) = class_instance(class_id, b"x"); + let expected_shape_id = shape_id(obj); crate::object::js_object_set_field(obj, 0, crate::JSValue::from_bits(1.0f64.to_bits())); crate::object::js_object_freeze(receiver); - let guard = - js_typed_feedback_class_field_set_guard(31, receiver, class_id, keys, key, 0, 2.0, 0); + let guard = js_typed_feedback_class_field_set_guard( + 31, + receiver, + class_id, + expected_shape_id, + key, + 0, + 2.0, + 0, + ); assert_eq!(guard, 0); assert_eq!( crate::object::js_object_get_field(obj, 0).bits(), @@ -1579,7 +1592,8 @@ fn typed_feedback_class_field_set_guard_falls_back_for_class_setter() { register(32, TypedFeedbackSiteKind::PropertySet, "obj.x="); let class_id = 0x7EED_0032; - let (obj, keys, key, receiver) = class_instance(class_id, b"x"); + let (obj, _, key, receiver) = class_instance(class_id, b"x"); + let expected_shape_id = shape_id(obj); crate::object::js_object_set_field(obj, 0, crate::JSValue::from_bits(1.0f64.to_bits())); unsafe { crate::object::js_register_class_setter( @@ -1590,8 +1604,16 @@ fn typed_feedback_class_field_set_guard_falls_back_for_class_setter() { ); } - let guard = - js_typed_feedback_class_field_set_guard(32, receiver, class_id, keys, key, 0, 7.0, 0); + let guard = js_typed_feedback_class_field_set_guard( + 32, + receiver, + class_id, + expected_shape_id, + key, + 0, + 7.0, + 0, + ); assert_eq!(guard, 0); js_typed_feedback_record_fallback_call(32); crate::object::js_object_set_field_by_name(obj, key, 7.0); @@ -1622,18 +1644,33 @@ fn typed_feedback_class_field_get_guard_falls_back_after_shape_transition() { register(39, TypedFeedbackSiteKind::PropertyGet, "obj.x"); let class_id = 0x7EED_0039; - let (obj, expected_keys, key_x, receiver) = class_instance(class_id, b"x"); + let (obj, original_keys, key_x, receiver) = class_instance(class_id, b"x"); + let expected_shape_id = shape_id(obj); crate::object::js_object_set_field(obj, 0, crate::JSValue::from_bits(5.0f64.to_bits())); - let first = - js_typed_feedback_class_field_get_guard(39, receiver, class_id, expected_keys, key_x, 0, 0); + let first = js_typed_feedback_class_field_get_guard( + 39, + receiver, + class_id, + expected_shape_id, + key_x, + 0, + 0, + ); assert_eq!(first, 1); let key_y = crate::string::js_string_from_bytes(b"y".as_ptr(), 1); crate::object::js_object_set_field_by_name(obj, key_y, 10.0); - assert_ne!(unsafe { (*obj).keys_array }, expected_keys); + assert_ne!(unsafe { (*obj).keys_array }, original_keys); - let second = - js_typed_feedback_class_field_get_guard(39, receiver, class_id, expected_keys, key_x, 0, 0); + let second = js_typed_feedback_class_field_get_guard( + 39, + receiver, + class_id, + expected_shape_id, + key_x, + 0, + 0, + ); assert_eq!(second, 0); js_typed_feedback_record_fallback_call(39); let stored = crate::object::js_object_get_field_by_name_f64(obj, key_x); @@ -1645,6 +1682,50 @@ fn typed_feedback_class_field_get_guard_falls_back_after_shape_transition() { assert_eq!(site.fallback_calls, 1); } +#[test] +fn typed_feedback_class_field_guard_ignores_object_header_shape_mirrors() { + let _guard = typed_feedback_test_lock(); + reset_typed_feedback_for_tests(); + register(8067, TypedFeedbackSiteKind::PropertyGet, "obj.x"); + + let class_id = 0x7EED_8067; + let (obj, original_keys, key_x, receiver) = class_instance(class_id, b"x"); + let expected_shape_id = shape_id(obj); + let original_field_count = unsafe { (*obj).field_count }; + + unsafe { + // These are ABI mirrors retained until the later header-shrink issue. + // An authoritative guard must not consult either one. + // GC_STORE_AUDIT(POINTER_FREE): test sabotage removes the compatibility edge by storing null. + (*obj).keys_array = std::ptr::null_mut(); + (*obj).field_count = 0; + } + let passed = js_typed_feedback_class_field_get_guard( + 8067, + receiver, + class_id, + expected_shape_id, + key_x, + 0, + 0, + ); + unsafe { + // GC_STORE_AUDIT(BARRIERED): restoring the saved compatibility edge is followed by the ordinary object-slot barrier. + (*obj).keys_array = original_keys; + crate::gc::runtime_write_barrier_slot( + obj as usize, + &(*obj).keys_array as *const _ as usize, + original_keys as u64, + ); + (*obj).field_count = original_field_count; + } + + assert_eq!(passed, 1, "guard must consume ShapeDescriptor facts"); + let site = &typed_feedback_snapshot().sites[0]; + assert_eq!(site.guard_passes, 1); + assert_eq!(site.guard_failures, 0); +} + #[test] fn typed_feedback_class_field_get_guard_requires_raw_f64_layout_when_requested() { let _guard = typed_feedback_test_lock(); @@ -1652,7 +1733,8 @@ fn typed_feedback_class_field_get_guard_requires_raw_f64_layout_when_requested() register(43, TypedFeedbackSiteKind::PropertyGet, "obj.x"); let class_id = 0x7EED_0043; - let (obj, expected_keys, key_x, receiver) = class_instance(class_id, b"x"); + let (obj, _, key_x, receiver) = class_instance(class_id, b"x"); + let expected_shape_id = shape_id(obj); crate::object::js_object_set_field(obj, 0, crate::JSValue::number(5.0)); let raw_mask = [0b1u64]; crate::gc::js_gc_init_typed_shape_layout( @@ -1665,8 +1747,15 @@ fn typed_feedback_class_field_get_guard_requires_raw_f64_layout_when_requested() ); crate::gc::test_reset_typed_raw_f64_descriptor_queries(); - let first = - js_typed_feedback_class_field_get_guard(43, receiver, class_id, expected_keys, key_x, 0, 1); + let first = js_typed_feedback_class_field_get_guard( + 43, + receiver, + class_id, + expected_shape_id, + key_x, + 0, + 1, + ); assert_eq!(first, 1); assert_eq!( crate::gc::test_typed_raw_f64_descriptor_queries(), @@ -1677,8 +1766,15 @@ fn typed_feedback_class_field_get_guard_requires_raw_f64_layout_when_requested() let payload = crate::string::js_string_from_bytes(b"boxed".as_ptr(), 5); crate::object::js_object_set_field(obj, 0, crate::JSValue::string_ptr(payload)); - let second = - js_typed_feedback_class_field_get_guard(43, receiver, class_id, expected_keys, key_x, 0, 1); + let second = js_typed_feedback_class_field_get_guard( + 43, + receiver, + class_id, + expected_shape_id, + key_x, + 0, + 1, + ); assert_eq!(second, 0); assert_eq!( crate::gc::test_typed_raw_f64_descriptor_queries(), @@ -1699,7 +1795,8 @@ fn typed_feedback_class_field_set_guard_requires_raw_f64_value_and_layout() { register(44, TypedFeedbackSiteKind::PropertySet, "obj.x="); let class_id = 0x7EED_0044; - let (obj, expected_keys, key_x, receiver) = class_instance(class_id, b"x"); + let (obj, _, key_x, receiver) = class_instance(class_id, b"x"); + let expected_shape_id = shape_id(obj); crate::object::js_object_set_field(obj, 0, crate::JSValue::number(1.0)); let raw_mask = [0b1u64]; crate::gc::js_gc_init_typed_shape_layout( @@ -1716,7 +1813,7 @@ fn typed_feedback_class_field_set_guard_requires_raw_f64_value_and_layout() { 44, receiver, class_id, - expected_keys, + expected_shape_id, key_x, 0, 2.0, @@ -1735,7 +1832,7 @@ fn typed_feedback_class_field_set_guard_requires_raw_f64_value_and_layout() { 44, receiver, class_id, - expected_keys, + expected_shape_id, key_x, 0, payload_value, @@ -1748,7 +1845,7 @@ fn typed_feedback_class_field_set_guard_requires_raw_f64_value_and_layout() { 44, receiver, class_id, - expected_keys, + expected_shape_id, key_x, 0, f64::from_bits(short.bits()), @@ -1761,7 +1858,7 @@ fn typed_feedback_class_field_set_guard_requires_raw_f64_value_and_layout() { 44, receiver, class_id, - expected_keys, + expected_shape_id, key_x, 0, handle_value, @@ -1836,7 +1933,8 @@ fn typed_feedback_method_direct_guard_passes_for_exact_registered_method() { register(61, TypedFeedbackSiteKind::MethodCall, "obj.m()"); let class_id = 0x7EED_0061; - let (_, keys, _, receiver) = class_instance(class_id, b"x"); + let (obj, _, _, receiver) = class_instance(class_id, b"x"); + let expected_shape_id = shape_id(obj); unsafe { register_test_method(class_id, b"m") }; let guard = unsafe { @@ -1844,7 +1942,7 @@ fn typed_feedback_method_direct_guard_passes_for_exact_registered_method() { 61, receiver, class_id, - keys, + expected_shape_id, b"m".as_ptr() as *const i8, 1, test_direct_method_ptr(), @@ -1866,7 +1964,8 @@ fn typed_feedback_method_direct_guard_fails_for_own_method_replacement() { register(62, TypedFeedbackSiteKind::MethodCall, "obj.m()"); let class_id = 0x7EED_0062; - let (obj, keys, _, receiver) = class_instance(class_id, b"x"); + let (obj, _, _, receiver) = class_instance(class_id, b"x"); + let expected_shape_id = shape_id(obj); unsafe { register_test_method(class_id, b"m") }; let key_m = crate::string::js_string_from_bytes(b"m".as_ptr(), 1); crate::object::js_object_set_field_by_name(obj, key_m, 123.0); @@ -1876,7 +1975,7 @@ fn typed_feedback_method_direct_guard_fails_for_own_method_replacement() { 62, receiver, class_id, - keys, + expected_shape_id, b"m".as_ptr() as *const i8, 1, test_direct_method_ptr(), @@ -1898,7 +1997,8 @@ fn typed_feedback_method_direct_guard_fails_for_prototype_method_registration() register(63, TypedFeedbackSiteKind::MethodCall, "obj.m()"); let class_id = 0x7EED_0063; - let (_, keys, _, receiver) = class_instance(class_id, b"x"); + let (obj, _, _, receiver) = class_instance(class_id, b"x"); + let expected_shape_id = shape_id(obj); unsafe { register_test_method(class_id, b"m"); crate::object::js_register_prototype_method( @@ -1914,7 +2014,7 @@ fn typed_feedback_method_direct_guard_fails_for_prototype_method_registration() 63, receiver, class_id, - keys, + expected_shape_id, b"m".as_ptr() as *const i8, 1, test_direct_method_ptr(), @@ -1943,7 +2043,7 @@ fn typed_feedback_method_direct_guard_fails_for_native_receiver() { 64, receiver, crate::object::NATIVE_MODULE_CLASS_ID, - std::ptr::null(), + shape_id(native), b"m".as_ptr() as *const i8, 1, test_direct_method_ptr(), @@ -1980,14 +2080,15 @@ fn typed_feedback_method_direct_guard_fails_after_megamorphic_site() { } let class_id = 0x7EED_0065; - let (_, keys, _, receiver) = class_instance(class_id, b"x"); + let (obj, _, _, receiver) = class_instance(class_id, b"x"); + let expected_shape_id = shape_id(obj); unsafe { register_test_method(class_id, b"m") }; let guard = unsafe { js_typed_feedback_method_direct_call_guard( 65, receiver, class_id, - keys, + expected_shape_id, b"m".as_ptr() as *const i8, 1, test_direct_method_ptr(), diff --git a/crates/perry-runtime/src/value/addr_class.rs b/crates/perry-runtime/src/value/addr_class.rs index da3a83a575..edd9debc73 100644 --- a/crates/perry-runtime/src/value/addr_class.rs +++ b/crates/perry-runtime/src/value/addr_class.rs @@ -119,6 +119,12 @@ pub fn is_stream_id_band(id: usize) -> bool { /// Check if a pointer is a valid heap object (safe to dereference GcHeader). /// Values below 0x100000 (1MB) are likely INT32_TAG extracts, small handles, /// or null. The upper bound filters out NaN-box tag bits that leaked through. +/// Linux-family AArch64 targets can map userspace arenas anywhere in the full +/// low 48-bit VA range, including addresses with bit 47 set (observed under +/// the native Linux ARM provider gate around `0x0000_e000_...`). Those +/// addresses are still exactly representable in Perry's 48-bit NaN-box +/// payload. The half-range bound used by x86-64 canonical low addresses must +/// not reject them. /// /// Issue #73 follow-up: raised the lower bound from 1 MB to 2 TB to reject /// corrupted NaN-boxes whose 48-bit handle lands in the 1-2 TB window @@ -181,7 +187,17 @@ pub(crate) fn is_valid_obj_ptr(ptr: *const u8) -> bool { target_os = "visionos", )))] const HEAP_MIN: u64 = 0x200_0000_0000; - (HEAP_MIN..0x8000_0000_0000).contains(&addr) + #[cfg(all( + target_arch = "aarch64", + any(target_os = "android", target_os = "linux") + ))] + const HEAP_MAX: u64 = 0x1_0000_0000_0000; + #[cfg(not(all( + target_arch = "aarch64", + any(target_os = "android", target_os = "linux") + )))] + const HEAP_MAX: u64 = 0x8000_0000_0000; + (HEAP_MIN..HEAP_MAX).contains(&addr) } /// True when `addr` is outside every handle band AND inside the platform @@ -426,4 +442,18 @@ mod tests { // this representative address. assert!(is_valid_obj_ptr(0x0000_000a_0000_0000usize as *const u8)); } + + #[cfg(all( + target_arch = "aarch64", + any(target_os = "android", target_os = "linux") + ))] + #[test] + fn linux_family_aarch64_accepts_the_full_low_48_bit_heap_range() { + // The provider-dylib regression allocated its first ObjectHeader in + // this half of the AArch64 userspace range. It remains a plain 48-bit + // NaN-box payload; only x86-64's canonical-address rule excludes it. + assert!(is_valid_obj_ptr(0x0000_e000_0000_1000usize as *const u8)); + assert!(is_plausible_heap_addr(0x0000_e000_0000_1000)); + assert!(!is_valid_obj_ptr(0x0001_0000_0000_0000usize as *const u8)); + } } diff --git a/experiments/llvm-inprocess-spike/batch_kernel.ll b/experiments/llvm-inprocess-spike/batch_kernel.ll index 3538c091de..6a80241a5b 100644 --- a/experiments/llvm-inprocess-spike/batch_kernel.ll +++ b/experiments/llvm-inprocess-spike/batch_kernel.ll @@ -50,7 +50,6 @@ module asm ".no_dead_strip __LLVM_StackMaps" @PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT = external global i32 @PERRY_TA_KIND_CACHE = external global [64 x i64] @PERRY_TA_VIEW_GUARD = external global i64 -@PERRY_IC_EPOCH = external global i64 @perry_class_keys_batch_ts__Row = internal global i64 0 @perry_class_keys_batch_ts____AnonShape_e75d7e32e87dc826 = internal global i64 0 @perry_class_keys_batch_ts____AnonShape_c33c4204c28b9430 = internal global i64 0 @@ -4747,7 +4746,7 @@ pic.token.33: %r152 = and i1 %r150, %r151 %r153 = getelementptr i64, ptr @perry_ic_2, i64 2 %r154 = load i64, ptr %r153 - %r155 = load i64, ptr @PERRY_IC_EPOCH + %r155 = add i64 0, 0 %r156 = icmp eq i64 %r154, %r155 %r157 = or i1 %r144, %r156 %r158 = and i1 %r152, %r157 @@ -5048,7 +5047,7 @@ pic.token.60: %r335 = and i1 %r333, %r334 %r336 = getelementptr i64, ptr @perry_ic_5, i64 2 %r337 = load i64, ptr %r336 - %r338 = load i64, ptr @PERRY_IC_EPOCH + %r338 = add i64 0, 0 %r339 = icmp eq i64 %r337, %r338 %r340 = or i1 %r327, %r339 %r341 = and i1 %r335, %r340 @@ -5304,7 +5303,7 @@ pic.token.14: %r50 = and i1 %r48, %r49 %r51 = getelementptr i64, ptr @perry_ic_8, i64 2 %r52 = load i64, ptr %r51 - %r53 = load i64, ptr @PERRY_IC_EPOCH + %r53 = add i64 0, 0 %r54 = icmp eq i64 %r52, %r53 %r55 = or i1 %r42, %r54 %r56 = and i1 %r50, %r55 @@ -5509,7 +5508,7 @@ pic.token.35: %r183 = and i1 %r181, %r182 %r184 = getelementptr i64, ptr @perry_ic_10, i64 2 %r185 = load i64, ptr %r184 - %r186 = load i64, ptr @PERRY_IC_EPOCH + %r186 = add i64 0, 0 %r187 = icmp eq i64 %r185, %r186 %r188 = or i1 %r175, %r187 %r189 = and i1 %r183, %r188 @@ -5881,7 +5880,7 @@ pic.token.70: %r378 = and i1 %r376, %r377 %r379 = getelementptr i64, ptr @perry_ic_12, i64 2 %r380 = load i64, ptr %r379 - %r381 = load i64, ptr @PERRY_IC_EPOCH + %r381 = add i64 0, 0 %r382 = icmp eq i64 %r380, %r381 %r383 = or i1 %r370, %r382 %r384 = and i1 %r378, %r383 @@ -6169,7 +6168,7 @@ pic.token.14: %r51 = and i1 %r49, %r50 %r52 = getelementptr i64, ptr @perry_ic_14, i64 2 %r53 = load i64, ptr %r52 - %r54 = load i64, ptr @PERRY_IC_EPOCH + %r54 = add i64 0, 0 %r55 = icmp eq i64 %r53, %r54 %r56 = or i1 %r43, %r55 %r57 = and i1 %r51, %r56 @@ -6347,7 +6346,7 @@ pic.token.33: %r177 = and i1 %r175, %r176 %r178 = getelementptr i64, ptr @perry_ic_16, i64 2 %r179 = load i64, ptr %r178 - %r180 = load i64, ptr @PERRY_IC_EPOCH + %r180 = add i64 0, 0 %r181 = icmp eq i64 %r179, %r180 %r182 = or i1 %r169, %r181 %r183 = and i1 %r177, %r182 @@ -6632,7 +6631,7 @@ pic.token.64: %r351 = and i1 %r349, %r350 %r352 = getelementptr i64, ptr @perry_ic_18, i64 2 %r353 = load i64, ptr %r352 - %r354 = load i64, ptr @PERRY_IC_EPOCH + %r354 = add i64 0, 0 %r355 = icmp eq i64 %r353, %r354 %r356 = or i1 %r343, %r355 %r357 = and i1 %r351, %r356 @@ -6830,7 +6829,7 @@ pic.token.85: %r482 = and i1 %r480, %r481 %r483 = getelementptr i64, ptr @perry_ic_20, i64 2 %r484 = load i64, ptr %r483 - %r485 = load i64, ptr @PERRY_IC_EPOCH + %r485 = add i64 0, 0 %r486 = icmp eq i64 %r484, %r485 %r487 = or i1 %r474, %r486 %r488 = and i1 %r482, %r487 @@ -7016,7 +7015,7 @@ pic.token.104: %r611 = and i1 %r609, %r610 %r612 = getelementptr i64, ptr @perry_ic_22, i64 2 %r613 = load i64, ptr %r612 - %r614 = load i64, ptr @PERRY_IC_EPOCH + %r614 = add i64 0, 0 %r615 = icmp eq i64 %r613, %r614 %r616 = or i1 %r603, %r615 %r617 = and i1 %r611, %r616 @@ -7194,7 +7193,7 @@ pic.token.123: %r737 = and i1 %r735, %r736 %r738 = getelementptr i64, ptr @perry_ic_24, i64 2 %r739 = load i64, ptr %r738 - %r740 = load i64, ptr @PERRY_IC_EPOCH + %r740 = add i64 0, 0 %r741 = icmp eq i64 %r739, %r740 %r742 = or i1 %r729, %r741 %r743 = and i1 %r737, %r742 @@ -7401,7 +7400,7 @@ pic.token.145: %r866 = and i1 %r864, %r865 %r867 = getelementptr i64, ptr @perry_ic_26, i64 2 %r868 = load i64, ptr %r867 - %r869 = load i64, ptr @PERRY_IC_EPOCH + %r869 = add i64 0, 0 %r870 = icmp eq i64 %r868, %r869 %r871 = or i1 %r858, %r870 %r872 = and i1 %r866, %r871 @@ -7579,7 +7578,7 @@ pic.token.164: %r992 = and i1 %r990, %r991 %r993 = getelementptr i64, ptr @perry_ic_28, i64 2 %r994 = load i64, ptr %r993 - %r995 = load i64, ptr @PERRY_IC_EPOCH + %r995 = add i64 0, 0 %r996 = icmp eq i64 %r994, %r995 %r997 = or i1 %r984, %r996 %r998 = and i1 %r992, %r997 @@ -7954,7 +7953,7 @@ pic.token.23: %r122 = and i1 %r120, %r121 %r123 = getelementptr i64, ptr @perry_ic_31, i64 2 %r124 = load i64, ptr %r123 - %r125 = load i64, ptr @PERRY_IC_EPOCH + %r125 = add i64 0, 0 %r126 = icmp eq i64 %r124, %r125 %r127 = or i1 %r114, %r126 %r128 = and i1 %r122, %r127 @@ -8245,7 +8244,7 @@ pic.token.50: %r341 = and i1 %r339, %r340 %r342 = getelementptr i64, ptr @perry_ic_34, i64 2 %r343 = load i64, ptr %r342 - %r344 = load i64, ptr @PERRY_IC_EPOCH + %r344 = add i64 0, 0 %r345 = icmp eq i64 %r343, %r344 %r346 = or i1 %r333, %r345 %r347 = and i1 %r341, %r346 @@ -8780,7 +8779,7 @@ pic.token.93: %r682 = and i1 %r680, %r681 %r683 = getelementptr i64, ptr @perry_ic_39, i64 2 %r684 = load i64, ptr %r683 - %r685 = load i64, ptr @PERRY_IC_EPOCH + %r685 = add i64 0, 0 %r686 = icmp eq i64 %r684, %r685 %r687 = or i1 %r674, %r686 %r688 = and i1 %r682, %r687 @@ -11859,7 +11858,7 @@ pic.token.39: %r214 = and i1 %r212, %r213 %r215 = getelementptr i64, ptr @perry_ic_60, i64 2 %r216 = load i64, ptr %r215 - %r217 = load i64, ptr @PERRY_IC_EPOCH + %r217 = add i64 0, 0 %r218 = icmp eq i64 %r216, %r217 %r219 = or i1 %r206, %r218 %r220 = and i1 %r214, %r219 @@ -12215,7 +12214,7 @@ pic.token.74: %r442 = and i1 %r440, %r441 %r443 = getelementptr i64, ptr @perry_ic_63, i64 2 %r444 = load i64, ptr %r443 - %r445 = load i64, ptr @PERRY_IC_EPOCH + %r445 = add i64 0, 0 %r446 = icmp eq i64 %r444, %r445 %r447 = or i1 %r434, %r446 %r448 = and i1 %r442, %r447 @@ -12486,7 +12485,7 @@ pic.token.99: %r631 = and i1 %r629, %r630 %r632 = getelementptr i64, ptr @perry_ic_66, i64 2 %r633 = load i64, ptr %r632 - %r634 = load i64, ptr @PERRY_IC_EPOCH + %r634 = add i64 0, 0 %r635 = icmp eq i64 %r633, %r634 %r636 = or i1 %r623, %r635 %r637 = and i1 %r631, %r636 diff --git a/experiments/llvm-inprocess-spike/eh_text.ll b/experiments/llvm-inprocess-spike/eh_text.ll index 2a7ad3ff39..e6a53132ec 100644 --- a/experiments/llvm-inprocess-spike/eh_text.ll +++ b/experiments/llvm-inprocess-spike/eh_text.ll @@ -48,7 +48,6 @@ module asm ".no_dead_strip __LLVM_StackMaps" @PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT = external global i32 @PERRY_TA_KIND_CACHE = external global [64 x i64] @PERRY_TA_VIEW_GUARD = external global i64 -@PERRY_IC_EPOCH = external global i64 @perry_ic_1 = private global [12 x i64] zeroinitializer @perry_ic_3 = private global [12 x i64] zeroinitializer @perry_ic_5 = private global [12 x i64] zeroinitializer @@ -3204,7 +3203,7 @@ pic.token.20: %r62 = and i1 %r60, %r61 %r63 = getelementptr i64, ptr @perry_ic_1, i64 2 %r64 = load i64, ptr %r63 - %r65 = load i64, ptr @PERRY_IC_EPOCH + %r65 = add i64 0, 0 %r66 = icmp eq i64 %r64, %r65 %r67 = or i1 %r54, %r66 %r68 = and i1 %r62, %r67 @@ -3891,7 +3890,7 @@ pic.token.24: %r65 = and i1 %r63, %r64 %r66 = getelementptr i64, ptr @perry_ic_3, i64 2 %r67 = load i64, ptr %r66 - %r68 = load i64, ptr @PERRY_IC_EPOCH + %r68 = add i64 0, 0 %r69 = icmp eq i64 %r67, %r68 %r70 = or i1 %r57, %r69 %r71 = and i1 %r65, %r70 @@ -4086,7 +4085,7 @@ pic.token.45: %r208 = and i1 %r206, %r207 %r209 = getelementptr i64, ptr @perry_ic_5, i64 2 %r210 = load i64, ptr %r209 - %r211 = load i64, ptr @PERRY_IC_EPOCH + %r211 = add i64 0, 0 %r212 = icmp eq i64 %r210, %r211 %r213 = or i1 %r200, %r212 %r214 = and i1 %r208, %r213 @@ -4968,7 +4967,7 @@ pic.token.35: %r108 = and i1 %r106, %r107 %r109 = getelementptr i64, ptr @perry_ic_7, i64 2 %r110 = load i64, ptr %r109 - %r111 = load i64, ptr @PERRY_IC_EPOCH + %r111 = add i64 0, 0 %r112 = icmp eq i64 %r110, %r111 %r113 = or i1 %r100, %r112 %r114 = and i1 %r108, %r113 @@ -5373,7 +5372,7 @@ pic.token.32: %r100 = and i1 %r98, %r99 %r101 = getelementptr i64, ptr @perry_ic_9, i64 2 %r102 = load i64, ptr %r101 - %r103 = load i64, ptr @PERRY_IC_EPOCH + %r103 = add i64 0, 0 %r104 = icmp eq i64 %r102, %r103 %r105 = or i1 %r92, %r104 %r106 = and i1 %r100, %r105 diff --git a/experiments/llvm-inprocess-spike/spike_text.ll b/experiments/llvm-inprocess-spike/spike_text.ll index 23a4299a0e..dcefe7d1c6 100644 --- a/experiments/llvm-inprocess-spike/spike_text.ll +++ b/experiments/llvm-inprocess-spike/spike_text.ll @@ -35,7 +35,6 @@ module asm ".no_dead_strip __LLVM_StackMaps" @PERRY_INCREMENTAL_MARK_BARRIER_ACTIVE_COUNT = external global i32 @PERRY_TA_KIND_CACHE = external global [64 x i64] @PERRY_TA_VIEW_GUARD = external global i64 -@PERRY_IC_EPOCH = external global i64 @perry_class_keys_spike_ts__Point = internal global i64 0 @spike_ts_.str.0.handle = internal global double 0.0 @spike_ts_.str.1.handle = internal global double 0.0 diff --git a/scripts/compiler_output_harness/repsel_census.py b/scripts/compiler_output_harness/repsel_census.py index 86b0e639ec..c0ea66bae9 100644 --- a/scripts/compiler_output_harness/repsel_census.py +++ b/scripts/compiler_output_harness/repsel_census.py @@ -62,6 +62,7 @@ from __future__ import annotations import argparse +from collections import Counter import json import os import re @@ -364,6 +365,11 @@ ("ptr-shape", "constructor argument", "rule 1 (provenance)"): 1, ("ptr-shape", "object literal property value", "rule 1 (provenance)"): 1, ("ptr-shape", "returned expression operand", "rule 1 (provenance)"): 1, + ( + "ptr-shape", + "returned expression operand", + "rule 1 (provenance) — already served by return-shape", + ): 2, ( "ptr-shape", "return", @@ -1411,6 +1417,14 @@ def self_test(_args: argparse.Namespace) -> int: ("constructor argument", "rule 1 (provenance)"), ("object literal property value", "rule 1 (provenance)"), ("returned expression operand", "rule 1 (provenance)"), + ( + "returned expression operand", + "rule 1 (provenance) — already served by return-shape", + ), + ( + "returned expression operand", + "rule 1 (provenance) — already served by return-shape", + ), ("return", "rule 1 (provenance) — already served by return-shape"), ) bucket_report = { @@ -1444,9 +1458,9 @@ def self_test(_args: argparse.Namespace) -> int: ], } buckets = census_from_report(bucket_report) - assert buckets["alloc_buckets"] == { - alloc_bucket_key("ptr-shape", c, r): 1 for c, r in bucket_rows - }, buckets["alloc_buckets"] + assert buckets["alloc_buckets"] == dict( + Counter(alloc_bucket_key("ptr-shape", c, r) for c, r in bucket_rows) + ), buckets["alloc_buckets"] # Named, not `next(iter(...))`: #7170 R1 added a second fixture to the # table, and an implicit "first entry" would have silently retargeted every @@ -1496,6 +1510,11 @@ def self_test(_args: argparse.Namespace) -> int: "returned expression operand", "rule 1 (provenance)", ): 1, + alloc_bucket_key( + "ptr-shape", + "returned expression operand", + "rule 1 (provenance) — already served by return-shape", + ): 2, alloc_bucket_key( "ptr-shape", "return", @@ -1523,11 +1542,13 @@ def self_test(_args: argparse.Namespace) -> int: # RULE present, but on the wrong POSITION. With the rule and the context # floored independently this passed; keyed as one tuple it cannot. swapped = json.loads(json.dumps(good)) - swapped[fixture]["alloc_buckets"] = { - alloc_bucket_key("ptr-shape", c, r): 1 - for c, r in bucket_rows - if c != "return" - } + swapped[fixture]["alloc_buckets"] = dict( + Counter( + alloc_bucket_key("ptr-shape", c, r) + for c, r in bucket_rows + if c != "return" + ) + ) swapped[fixture]["alloc_buckets"][ alloc_bucket_key( "ptr-shape", diff --git a/scripts/gc_runtime_root_holders.json b/scripts/gc_runtime_root_holders.json index ed32a5b654..9727515c6e 100644 --- a/scripts/gc_runtime_root_holders.json +++ b/scripts/gc_runtime_root_holders.json @@ -80,12 +80,6 @@ "scanner": "object::scan_class_side_table_roots_mut and its budgeted step twin (class_registry/gc_roots.rs:138 and :256)", "why": "The class side tables are declared in state.rs and scanned from gc_roots.rs. Both twins visit it \u2014 #7239 diffed all eight budgeted (FULL, STEP) pairs and found no drift." }, - { - "file": "crates/perry-runtime/src/object/field_get_set/ic_miss.rs", - "name": "PERRY_IC_EPOCH", - "verdict": "not_a_gc_pointer", - "why": "Inline-cache invalidation epoch. Epoch invalidation is the third rooting strategy this tree uses (see object/prop_plan.rs) \u2014 the cache is discarded on bump rather than scanned." - }, { "file": "crates/perry-runtime/src/process.rs", "name": "MODULE_LOADER_NEXT_RESOLVE", diff --git a/scripts/raw_handle_debt.py b/scripts/raw_handle_debt.py index 1d494b5a0c..278e3566ed 100755 --- a/scripts/raw_handle_debt.py +++ b/scripts/raw_handle_debt.py @@ -8,8 +8,11 @@ ALREADY; what was missing was ordering the re-read against the collection point. `RuntimeHandle::across_{mut,const,nanbox}` expresses that ordering in one call -and never binds the pre-call address. Each bare `get_raw_*_ptr` is a site where -that ordering is a review question instead of a shape. +and never binds the pre-call address. `with_{mut,const}_ptr` covers the other +legitimate shape: passing the current pointer directly to a non-allocating +operation or to an entry point that establishes its own root before it can +allocate. Each bare `get_raw_*_ptr` is a site where those contracts are a +review question instead of a shape. This is a DEBT COUNTER, not a soundness proof. Rust has no effect system to mark "this call may allocate", so no signature can reject holding a stale copy. Not @@ -42,7 +45,7 @@ BASELINE = ROOT / "scripts" / "raw_handle_debt_baseline.txt" PAT = re.compile(r"\.get_raw_(?:mut|const)_ptr\b") -# The accessors and the `across_*` combinators are DEFINED here and call each +# The accessors and scoped-pointer combinators are DEFINED here and call each # other; counting this file would make the ratchet count its own implementation # and rise every time a combinator is added. Exclude it. EXCLUDE = {"crates/perry-runtime/src/gc/roots/runtime_handles.rs"} @@ -89,7 +92,8 @@ def check_per_module(per_file): if path not in ceilings: bad.append( f"{path}: {n} bare read(s) in a module with no ceiling. New code must " - f"use RuntimeHandle::across_{{mut,const,nanbox}}; see #7341." + f"use RuntimeHandle::across_{{mut,const,nanbox}} or " + f"with_{{mut,const}}_ptr; see #7341." ) elif n > ceilings[path]: bad.append(f"{path}: {n} bare reads exceeds its ceiling of {ceilings[path]}") @@ -133,7 +137,8 @@ def compare_across_base(base_total, base_ceilings, head_total, head_ceilings): bad.append( f"baseline raised {base_total} -> {head_total} relative to the merge " f"base. The ratchet only goes down; convert the new sites to " - f"RuntimeHandle::across_{{mut,const,nanbox}} instead of recording them." + f"RuntimeHandle::across_{{mut,const,nanbox}} / " + f"with_{{mut,const}}_ptr instead of recording them." ) for path, ceiling in sorted(head_ceilings.items()): was = base_ceilings.get(path, 0) @@ -208,6 +213,8 @@ def self_test(): must_not_match = [ "let (found, obj) = h.across_mut::(|| f());", "h.across_const::(|| g())", + "h.with_mut_ptr::(|obj| consume(obj))", + "h.with_const_ptr::(|key| lookup(key))", "h.get_nanbox_f64()", ] for line in must_match: @@ -303,7 +310,7 @@ def main(): prev = int(BASELINE.read_text().split()[0]) if BASELINE.exists() else None if prev is not None and total > prev: print(f"refusing to raise the baseline: {prev} -> {total}") - print("the ratchet only goes down; convert sites to across_* instead") + print("the ratchet only goes down; convert sites to across_*/with_* instead") return 1 BASELINE.write_text(f"{total}\n") # Rewrite the per-module ceilings too, preserving the header. Entries @@ -337,17 +344,18 @@ def main(): print(f"::error::per-module raw-handle rules: {len(module_violations)} violation(s)") for b in module_violations: print(f" {b}") - print("Use RuntimeHandle::across_{mut,const,nanbox} -- it runs the") - print("allocating call and returns the post-collection address, so the") - print("stale pointer is never bound. See #7341 and the header of") - print("scripts/raw_handle_debt_files.txt.") + print("Use RuntimeHandle::across_{mut,const,nanbox} for a post-call") + print("reload, or with_{mut,const}_ptr for a scoped argument to a") + print("non-allocating operation / self-rooting runtime entry point.") + print("See #7341 and scripts/raw_handle_debt_files.txt.") return 1 if total > prev: print(f"::error::raw-handle debt rose {prev} -> {total}") - print("Use RuntimeHandle::across_{mut,const,nanbox} -- it runs the") - print("allocating call and returns the post-collection address, so the") - print("stale pointer is never bound. See #7341.") + print("Use RuntimeHandle::across_{mut,const,nanbox} for a post-call") + print("reload, or with_{mut,const}_ptr for a scoped argument to a") + print("non-allocating operation / self-rooting runtime entry point.") + print("See #7341.") for path, n in sorted(per_file.items(), key=lambda kv: -kv[1])[:10]: print(f" {n:4d} {path}") return 1 diff --git a/scripts/raw_handle_debt_files.txt b/scripts/raw_handle_debt_files.txt index 0e60b90833..ec38c12d7f 100644 --- a/scripts/raw_handle_debt_files.txt +++ b/scripts/raw_handle_debt_files.txt @@ -6,8 +6,11 @@ # three directions are closed: # # 1. A file NOT listed here must have ZERO bare reads. New code is therefore -# clean by construction -- `RuntimeHandle::across_{mut,const,nanbox}` is -# the only way in, which is what "non-optional" means for #7341's +# clean by construction. `RuntimeHandle::across_{mut,const,nanbox}` pairs +# an allocating call with its post-call reload; `with_{mut,const}_ptr` +# scopes an argument passed to a non-allocating operation or a runtime +# entry point that establishes its own root before allocation. Those are +# the only ways in, which is what "non-optional" means for #7341's # discipline (engine plan layer 3). # 2. A listed file must not EXCEED its ceiling. # 3. A listed file that reaches ZERO must be DELETED from this list. An entry diff --git a/scripts/shape_descriptor_census.py b/scripts/shape_descriptor_census.py index 47a386258e..94d2e7d06b 100644 --- a/scripts/shape_descriptor_census.py +++ b/scripts/shape_descriptor_census.py @@ -33,8 +33,13 @@ def rust_sources() -> dict[str, str]: } -def strip_rust_comments_and_literals(source: str) -> str: - """Blank comments/string literals while preserving code and newlines.""" +def _rust_without_comments_and_literals(source: str, preserve_offsets: bool) -> str: + """Blank comments/literals, optionally preserving every source offset.""" + + def blank_span(span: str) -> str: + if preserve_offsets: + return "".join("\n" if char == "\n" else " " for char in span) + return " " + "\n" * span.count("\n") chunks: list[str] = [] pos = 0 @@ -45,11 +50,11 @@ def strip_rust_comments_and_literals(source: str) -> str: if lexeme == "//": newline = source.find("\n", end) if newline < 0: - chunks.append(" ") - pos = len(source) - break - chunks.append("\n") - pos = newline + 1 + end = len(source) + else: + end = newline + 1 + chunks.append(blank_span(source[match.start() : end])) + pos = end continue if lexeme == "/*": depth = 1 @@ -69,12 +74,24 @@ def strip_rust_comments_and_literals(source: str) -> str: else: tail = QUOTED_STRING_TAIL.match(source, end) end = len(source) if tail is None else tail.end() - chunks.append(" " + "\n" * source[match.start() : end].count("\n")) + chunks.append(blank_span(source[match.start() : end])) pos = end chunks.append(source[pos:]) return "".join(chunks) +def strip_rust_comments_and_literals(source: str) -> str: + """Blank comments/string literals while preserving code and newlines.""" + + return _rust_without_comments_and_literals(source, preserve_offsets=False) + + +def blank_rust_comments_and_literals(source: str) -> str: + """Blank comments/literals while preserving every source-string offset.""" + + return _rust_without_comments_and_literals(source, preserve_offsets=True) + + def stripped_sources(sources: dict[str, str]) -> dict[str, str]: return {path: strip_rust_comments_and_literals(text) for path, text in sources.items()} @@ -90,6 +107,15 @@ def run_literal_lexer_selftest() -> None: clean = strip_rust_comments_and_literals(fixture) if len(re.findall(r"\.\s*keys_array\b", clean)) != 1: raise CensusError("literal lexer swallowed a real member between quote-char literals") + brace_fixture = '''fn brace_fixture() { + let string_brace = "}"; + let char_brace = '{'; + // } must not close the function + let live_after_literal_braces = 1; + } + ''' + if "live_after_literal_braces" not in function_body(brace_fixture, "brace_fixture"): + raise CensusError("raw function-body extraction counted literal/comment braces") def normalize_line(line: str) -> str: @@ -154,17 +180,18 @@ def observed_census(sources: dict[str, str]) -> dict[str, object]: def function_body(source: str, name: str) -> str: - match = re.search(rf"\bfn\s+{re.escape(name)}\b", source) + blanked = blank_rust_comments_and_literals(source) + match = re.search(rf"\bfn\s+{re.escape(name)}\b", blanked) if not match: raise CensusError(f"missing function body: {name}") - start = source.find("{", match.end()) + start = blanked.find("{", match.end()) if start < 0: raise CensusError(f"missing opening brace: {name}") depth = 0 - for i in range(start, len(source)): - if source[i] == "{": + for i in range(start, len(blanked)): + if blanked[i] == "{": depth += 1 - elif source[i] == "}": + elif blanked[i] == "}": depth -= 1 if depth == 0: return source[start + 1 : i] @@ -190,6 +217,18 @@ def assert_authority_surfaces(sources: dict[str, str]) -> None: "crates/perry-codegen/src/lower_call/new_alloc.rs", "crates/perry-runtime/src/gc/layout_slot_visit.rs", "crates/perry-runtime/src/object/field_set_by_name/tail.rs", + "crates/perry-runtime/src/typed_feedback/guards.rs", + "crates/perry-runtime/src/object/native_call_method.rs", + "crates/perry-runtime/src/object/exotic_expando.rs", + "crates/perry-runtime/src/object/field_get_set/get_field_by_name_tail.rs", + "crates/perry-runtime/src/object/field_get_set/ic_miss.rs", + "crates/perry-runtime/src/proxy/put_value.rs", + "crates/perry-runtime/src/gc/types.rs", + "crates/perry-runtime/src/regex.rs", + "crates/perry-codegen/src/expr/class_field_inline_guard.rs", + "crates/perry-codegen/src/expr/element_shape_guard.rs", + "crates/perry-codegen/src/expr/property_get/generic_dispatch.rs", + "crates/perry-codegen/src/expr/proxy_reflect.rs", ) missing = [path for path in authority_paths if path not in sources] if missing: @@ -204,11 +243,47 @@ def assert_authority_surfaces(sources: dict[str, str]) -> None: transition_tail = clean[ "crates/perry-runtime/src/object/field_set_by_name/tail.rs" ] + typed_guards = clean["crates/perry-runtime/src/typed_feedback/guards.rs"] + native_call_method = clean[ + "crates/perry-runtime/src/object/native_call_method.rs" + ] + exotic_expando = clean["crates/perry-runtime/src/object/exotic_expando.rs"] + get_field_tail = clean[ + "crates/perry-runtime/src/object/field_get_set/get_field_by_name_tail.rs" + ] + ic_miss = clean["crates/perry-runtime/src/object/field_get_set/ic_miss.rs"] + put_value = clean["crates/perry-runtime/src/proxy/put_value.rs"] + gc_types = clean["crates/perry-runtime/src/gc/types.rs"] + regex_runtime = clean["crates/perry-runtime/src/regex.rs"] + class_guard = clean[ + "crates/perry-codegen/src/expr/class_field_inline_guard.rs" + ] + element_guard = clean[ + "crates/perry-codegen/src/expr/element_shape_guard.rs" + ] + generic_pic = clean[ + "crates/perry-codegen/src/expr/property_get/generic_dispatch.rs" + ] + write_pics = clean["crates/perry-codegen/src/expr/proxy_reflect.rs"] + # Emitted ObjectHeader offsets and fail-closed constants are represented as + # Rust string literals, so inspect raw function bodies for these checks. + raw_class_guard = sources[ + "crates/perry-codegen/src/expr/class_field_inline_guard.rs" + ] + raw_element_guard = sources[ + "crates/perry-codegen/src/expr/element_shape_guard.rs" + ] + raw_generic_pic = sources[ + "crates/perry-codegen/src/expr/property_get/generic_dispatch.rs" + ] + raw_write_pics = sources["crates/perry-codegen/src/expr/proxy_reflect.rs"] for pattern, label in ( (r"descriptors\s*:\s*HashMap\s*<\s*u32\s*,\s*ShapeDescriptor", "by-id descriptor table"), (r"logical_key_count\s*:\s*u32", "exact logical-key fact"), (r"live_inline_slot_count\s*:\s*u32", "exact live-slot fact"), + (r"semantic_generation\s*:\s*u64", "semantic transition fact"), + (r"object_kind\s*:\s*ShapeObjectKind", "authoritative receiver-kind fact"), (r"\bfn\s+shape_descriptor_by_id\b", "by-id lookup"), (r"\bfn\s+debug_assert_object_shape_parity\b", "parity assertion"), (r"\bfn\s+synchronize_live_object_shape_descriptor_after_header_visit\b", "live-object descriptor mirror"), @@ -218,9 +293,15 @@ def assert_authority_surfaces(sources: dict[str, str]) -> None: allocator = function_body(shapes, "alloc_shape_id_from") require_code(allocator, r"\bcompare_exchange_weak\s*\(", "exhaustion park") - if re.search(r"\bfetch_add\s*\(|\bprocess\s*::\s*(?:abort|exit)\s*\(", allocator): - raise CensusError("ShapeId exhaustion is wrapping or unrecoverable") - require_code(shapes, r"\.unwrap_or\s*\(\s*0\s*\)", "recoverable exhaustion fallback") + if re.search(r"\bfetch_add\s*\(", allocator): + raise CensusError("ShapeId allocator wraps instead of parking") + require_code(shapes, r"\bfn\s+shape_id_exhausted_abort\b", "exhaustion fail-stop") + public_ensure = function_body(shapes, "shape_id_for_keys_ensure") + require_code( + public_ensure, + r"publish_shape_result\s*\(", + "typed shape-mint errors fail stop", + ) scanner = function_body(shapes, "scan_shape_table_rekey_mut") require_code(scanner, r"\bvisit_metadata_usize_slot\s*\(", "weak metadata rewrite") @@ -250,28 +331,30 @@ def assert_authority_surfaces(sources: dict[str, str]) -> None: "descriptor fact capture exact array type", ) - ensure = function_body(shapes, "shape_descriptor_ensure") + ensure = function_body(shapes, "shape_descriptor_ensure_with_generation") assert_before( ensure, "inner.descriptors.insert", - "inner.ids_by_facts.insert", + "inner.ids_by_facts.entry", "by-id descriptor before reverse accelerator", ) - sync = function_body(shapes, "synchronize_object_shape_descriptor") + sync = function_body(shapes, "synchronize_object_shape_descriptor_from") assert_before( sync, "shape_descriptor_ensure", "(*obj).parent_class_id = id", "descriptor before ObjectHeader ShapeId", ) - retirement = function_body(shapes, "retire_key_count_versions") + retirement = function_body(shapes, "retain_key_count_versions") require_code( retirement, r"ids_by_keys\s*\.\s*remove\s*\(\s*&keys\s*\)", - "keys-scoped descriptor retirement index", + "keys-scoped descriptor lineage index", ) if re.search(r"descriptors\s*\.\s*(?:iter|values|keys)\s*\(", retirement): - raise CensusError("shape descriptor retirement scans the global descriptor table") + raise CensusError("shape descriptor lineage repair scans the global descriptor table") + if "descriptors.remove" in retirement: + raise CensusError("live-key lineage repair eagerly deletes published descriptors") for name in ("shape_keys_grown", "shape_drop"): if "descriptors.remove" in function_body(shapes, name): raise CensusError(f"{name} eagerly deletes a sibling descriptor") @@ -293,6 +376,136 @@ def assert_authority_surfaces(sources: dict[str, str]) -> None: "transition-cache count before value", ) + # Runtime guard contracts may consume ShapeId/descriptor facts, never the + # compatibility ObjectHeader mirrors or a keys-pointer token. + for name in ( + "method_direct_call_contract", + "class_field_get_contract", + "class_field_fast_contract", + "class_field_set_contract", + ): + body = function_body(typed_guards, name) + if re.search(r"expected_keys|\(\s*\*\s*obj\s*\)\s*\.\s*(?:keys_array|field_count|object_type)\b", body): + raise CensusError(f"{name} reintroduced a legacy header guard fact") + require_code( + body, + r"object_shape(?:_(?:id|descriptor))?\s*\(", + f"{name} ShapeId authority", + ) + + for name in ("class_vtable_fast_guard", "js_native_call_method"): + body = function_body(native_call_method, name) + if re.search( + r"\(\s*\*\s*obj\s*\)\s*\.\s*(?:keys_array|field_count|object_type)\b|js_array_length\s*\(\s*keys\s*\)", + body, + ): + raise CensusError(f"{name} reintroduced a legacy method guard fact") + require_code( + body, + r"object_shape_descriptor\s*\(", + f"{name} ShapeId descriptor authority", + ) + require_code( + body, + r"logical_key_count\b", + f"{name} exact logical key count", + ) + + # RegExp identity lives in the GcHeader kind. No ObjectHeader payload word + # or registry/magic conjunction may decide these ordinary-object forks. + for name in ("object_is_regular", "object_is_shaped"): + body = function_body(object_mod, name) + require_code(body, r"obj_type\s*==\s*crate::gc::GC_TYPE_OBJECT", f"{name} GC kind") + if re.search(r"regex_header_has_magic|object_type", body): + raise CensusError(f"{name} reintroduced an old payload discriminator") + regexp_alloc = function_body(regex_runtime, "js_regexp_new") + require_code( + regexp_alloc, + r"gc_malloc\s*\([^;]*crate::gc::GC_TYPE_REGEXP", + "RegExp dedicated GC birth kind", + ) + expando_kind = function_body(exotic_expando, "exotic_expando_kind") + require_code( + expando_kind, + r"crate::gc::GC_TYPE_REGEXP\s*=>\s*Some\s*\(\s*ExoticKind::RegExp", + "RegExp expando dedicated kind", + ) + regexp_get = function_body(get_field_tail, "get_field_by_name_object_tail") + require_code( + regexp_get, + r"gc_type\s*==\s*crate::gc::GC_TYPE_REGEXP", + "RegExp property dispatch dedicated kind", + ) + if re.search( + r"GC_TYPE_OBJECT[^{};]*is_regex_pointer|is_regex_pointer[^{};]*GC_TYPE_OBJECT", + expando_kind + regexp_get, + ): + raise CensusError("RegExp dispatch reintroduced the former object-kind probe") + + read_miss = function_body(ic_miss, "js_object_get_field_ic_miss") + for body, label in ( + (read_miss, "read PIC miss"), + (function_body(put_value, "js_put_value_set_ic_miss"), "static write PIC miss"), + (function_body(put_value, "dyn_ic_try_store"), "dynamic write PIC hit"), + (function_body(put_value, "js_put_value_set_dyn_ic_miss"), "dynamic write PIC miss"), + ): + if re.search(r"else\s*\{\s*(?:keys|\(\s*\*\s*obj\s*\)\.keys_array)\s+as\s+u64", body): + raise CensusError(f"{label} reintroduced a keys-pointer token") + + # Emitted guards must not read the three payload offsets #8047 will remove. + for source, names in ( + (raw_class_guard, ( + "emit_class_field_loop_preheader_check", + "emit_proven_shape_recheck", + "emit_class_field_inline_precheck", + )), + (raw_element_guard, ("emit_element_shape_field_load",)), + ): + for name in names: + body = function_body(source, name) + if re.search(r"expected_keys|add\s*\([^\n]*\"(?:0|12|16)\"", body): + raise CensusError(f"{name} emits a removed ObjectHeader fact") + + generic_body = function_body(raw_generic_pic, "lower_generic_property_get") + if re.search(r"add\s*\(\s*I64\s*,\s*&obj_handle\s*,\s*\"(?:12|16)\"", generic_body): + raise CensusError("generic read PIC emits a removed ObjectHeader fact") + require_code( + generic_body, + r"select\s*\(\s*I1\s*,\s*&is_stamp\s*,\s*I64\s*,\s*&id_token\s*,\s*\"0\"\s*\)", + "generic read PIC invalid-id fail-closed token", + ) + for name in ("lower_put_value_static_write_ic", "lower_put_value_dyn_ic_inline"): + body = function_body(raw_write_pics, name) + if re.search(r"add\s*\(\s*I64\s*,\s*&(safe_target|t_handle)\s*,\s*\"(?:12|16)\"", body): + raise CensusError(f"{name} emits a removed ObjectHeader fact") + + require_code(gc_types, r"GC_TYPE_REGEXP\s*:\s*u8", "RegExp external discriminator") + regexp_info_match = re.search( + r"gc_type_info_entry\(\s*GC_TYPE_REGEXP\b[\s\S]*?\n\s*\)\s*\)", + gc_types, + ) + if not regexp_info_match: + raise CensusError("shape descriptor authority surface missing: RegExp type metadata") + regexp_info = regexp_info_match.group(0) + require_code( + regexp_info, + r"GcMoveHookKind::RegExpSideTables", + "RegExp address-owned relocation hook", + ) + require_code( + regexp_info, + r"GcFinalizeHookKind::RegExpSideTables", + "RegExp malloc-finalize side-table hook", + ) + if "OBJ_FLAG_CLASS_OBJECT" in gc_types + class_guard + element_guard + write_pics: + raise CensusError("class kind reintroduced a GcHeader layout-bit alias") + class_probe = function_body(object_mod, "object_is_regular") + require_code( + class_probe, + r"ShapeObjectKind::Ordinary", + "ordinary-object descriptor kind authority", + ) + def swap_once(source: str, left: str, right: str) -> str: left_at = source.find(left) @@ -367,11 +580,11 @@ def run_sabotage_selftests(sources: dict[str, str], baseline: dict[str, object]) inverted_publication = dict(sources) path = "crates/perry-runtime/src/object/shapes.rs" publication_body = function_body( - inverted_publication[path], "synchronize_object_shape_descriptor" + inverted_publication[path], "synchronize_object_shape_descriptor_from" ) inverted_body = swap_once( publication_body, - "shape_descriptor_ensure(keys, key_count, (*obj).field_count)", + "shape_descriptor_ensure_with_generation(", "(*obj).parent_class_id = id", ) inverted_publication[path] = inverted_publication[path].replace( @@ -385,7 +598,7 @@ def run_sabotage_selftests(sources: dict[str, str], baseline: dict[str, object]) unscoped_retirement = dict(sources) path = "crates/perry-runtime/src/object/shapes.rs" retirement_body = function_body( - unscoped_retirement[path], "retire_key_count_versions" + unscoped_retirement[path], "retain_key_count_versions" ) unscoped_body, substitutions = re.subn( r"ids_by_keys\s*\.\s*remove\s*\(\s*&keys\s*\)", @@ -403,6 +616,22 @@ def run_sabotage_selftests(sources: dict[str, str], baseline: dict[str, object]) lambda: assert_authority_surfaces(unscoped_retirement), ) + legacy_ir = dict(sources) + path = "crates/perry-codegen/src/expr/property_get/generic_dispatch.rs" + legacy_body, substitutions = re.subn( + r'add\(I64, &obj_handle, "8"\)', + 'add(I64, &obj_handle, "16")', + legacy_ir[path], + count=1, + ) + if substitutions != 1: + raise CensusError("legacy emitted-offset sabotage fixture missing") + legacy_ir[path] = legacy_body + expect_rejected( + "legacy keys-header offset in emitted PIC", + lambda: assert_authority_surfaces(legacy_ir), + ) + stale_summary = json.loads(json.dumps(baseline)) stale_summary["summary"]["raw_member_files"] += 1 expect_rejected( diff --git a/scripts/shape_descriptor_census_baseline.json b/scripts/shape_descriptor_census_baseline.json index 4e44c1e72d..0ed4761541 100644 --- a/scripts/shape_descriptor_census_baseline.json +++ b/scripts/shape_descriptor_census_baseline.json @@ -23,7 +23,6 @@ "crates/perry-codegen/src/target_layout.rs|pub fn object_header_size_bytes(target_triple: &str) -> u64 {": 1 }, "raw_member_callsite_multiset": { - "crates/perry-codegen/src/expr/property_get/generic_dispatch.rs|field_count|declaration|fn emit_slot_in_bounds(ctx: &mut FnCtx<'_>, slot: &str, field_count: &str) -> String {": 1, "crates/perry-codegen/src/lower_call/typed_shape_init.rs|field_count|declaration|field_count: u32,": 1, "crates/perry-codegen/tests/native_proof_regressions.rs|field_count|declaration|let loop_body = |field_count: usize| {": 1, "crates/perry-ext-events/src/lib.rs|field_count|declaration|fn js_object_alloc(class_id: u32, field_count: u32) -> *mut ObjectHeader;": 1, @@ -35,7 +34,6 @@ "crates/perry-ffi/src/types.rs|field_count|declaration|pub field_count: u32,": 1, "crates/perry-ffi/src/types.rs|keys_array|declaration|pub keys_array: *mut ArrayHeader,": 1, "crates/perry-ffi/src/types.rs|object_type|declaration|pub object_type: u32,": 1, - "crates/perry-runtime/src/array/element_shape.rs|object_type|access|if (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR {": 1, "crates/perry-runtime/src/builtins/console.rs|keys_array|access|let keys_array = (*obj_ptr).keys_array;": 2, "crates/perry-runtime/src/builtins/formatting.rs|keys_array|access|let _keys_array = (*obj_ptr).keys_array;": 1, "crates/perry-runtime/src/builtins/formatting.rs|keys_array|access|let keys_array = (*obj_ptr).keys_array;": 2, @@ -55,16 +53,15 @@ "crates/perry-runtime/src/error.rs|object_type|access|(*ptr).object_type = OBJECT_TYPE_ERROR;": 1, "crates/perry-runtime/src/error.rs|object_type|declaration|pub object_type: u32,": 1, "crates/perry-runtime/src/fs/dirent.rs|keys_array|access|let keys = (*obj_ptr).keys_array;": 1, - "crates/perry-runtime/src/gc/heap_snapshot.rs|field_count|access|let fc = unsafe { (*obj).field_count } as usize;": 1, - "crates/perry-runtime/src/gc/heap_snapshot.rs|keys_array|access|let keys_bits = (*obj).keys_array as u64;": 1, - "crates/perry-runtime/src/gc/layout.rs|field_count|access|if slot_index < (*object).field_count as usize {": 1, - "crates/perry-runtime/src/gc/layout.rs|field_count|access|let field_count = (*(user_ptr as *const crate::object::ObjectHeader)).field_count as usize;": 1, - "crates/perry-runtime/src/gc/layout.rs|field_count|access|let object_slot_count = (*obj_header).field_count as usize;": 1, - "crates/perry-runtime/src/gc/layout.rs|keys_array|access|(*(user_ptr as *const crate::object::ObjectHeader)).keys_array as usize": 1, - "crates/perry-runtime/src/gc/layout.rs|keys_array|access|let keys = (*obj_header).keys_array as usize;": 1, - "crates/perry-runtime/src/gc/layout_slot_visit.rs|field_count|access|let live_inline_slot_count = (*obj).field_count;": 1, + "crates/perry-runtime/src/gc/heap_snapshot.rs|field_count|access|.unwrap_or((*obj).field_count as usize)": 1, + "crates/perry-runtime/src/gc/heap_snapshot.rs|keys_array|access|.unwrap_or((*obj).keys_array as u64);": 1, + "crates/perry-runtime/src/gc/layout.rs|field_count|access|.unwrap_or((*obj_header).field_count as usize);": 1, + "crates/perry-runtime/src/gc/layout.rs|field_count|access|.unwrap_or((*object).field_count as usize);": 2, + "crates/perry-runtime/src/gc/layout.rs|keys_array|access|.unwrap_or((*obj_header).keys_array as usize);": 1, + "crates/perry-runtime/src/gc/layout.rs|keys_array|access|.unwrap_or((*object).keys_array as usize)": 1, + "crates/perry-runtime/src/gc/layout_slot_visit.rs|field_count|access|.unwrap_or((*obj).field_count);": 1, + "crates/perry-runtime/src/gc/layout_slot_visit.rs|keys_array|access|.unwrap_or((*obj).keys_array);": 1, "crates/perry-runtime/src/gc/layout_slot_visit.rs|keys_array|access|let new_keys = (*obj).keys_array as u64;": 1, - "crates/perry-runtime/src/gc/layout_slot_visit.rs|keys_array|access|let old_keys = (*obj).keys_array;": 1, "crates/perry-runtime/src/gc/tests/alloc.rs|object_type|declaration|object_type: crate::error::OBJECT_TYPE_ERROR,": 1, "crates/perry-runtime/src/gc/tests/barrier.rs|field_count|access|for i in 0..field_count as usize {": 2, "crates/perry-runtime/src/gc/tests/barrier.rs|field_count|declaration|unsafe fn field_index_not_on_last_page(fields: *mut u64, field_count: u32) -> usize {": 1, @@ -93,6 +90,9 @@ "crates/perry-runtime/src/gc/tests/runtime_roots/json_shape_template.rs|keys_array|access|let keys = (*obj).keys_array;": 1, "crates/perry-runtime/src/gc/tests/runtime_roots/transient_handles.rs|keys_array|access|assert!(!(*obj_after).keys_array.is_null());": 1, "crates/perry-runtime/src/gc/tests/runtime_roots/transient_handles.rs|keys_array|access|let key_value = crate::array::js_array_get((*obj_after).keys_array, 0).bits();": 1, + "crates/perry-runtime/src/gc/tests/shape_descriptor_authority.rs|field_count|access|(*obj).field_count = 0;": 1, + "crates/perry-runtime/src/gc/tests/shape_descriptor_authority.rs|keys_array|access|(*obj).keys_array = std::ptr::null_mut();": 1, + "crates/perry-runtime/src/gc/tests/shape_descriptor_authority.rs|keys_array|access|assert_eq!((*obj).keys_array as u64, descriptor.keys);": 1, "crates/perry-runtime/src/gc/tests/support.rs|field_count|access|(*obj).field_count = field_count;": 2, "crates/perry-runtime/src/gc/tests/support.rs|field_count|access|for i in 0..field_count as usize {": 2, "crates/perry-runtime/src/gc/tests/support.rs|field_count|declaration|field_count: u32,": 2, @@ -133,7 +133,6 @@ "crates/perry-runtime/src/json/stringify_tojson_probe.rs|keys_array|access|let keys = (*proto).keys_array;": 1, "crates/perry-runtime/src/json_tape_tests.rs|field_count|access|(*nested).field_count,": 1, "crates/perry-runtime/src/json_tape_tests.rs|field_count|access|(*object).field_count,": 2, - "crates/perry-runtime/src/native_abi.rs|object_type|access|let is_regular = (*obj).object_type == crate::error::OBJECT_TYPE_REGULAR;": 1, "crates/perry-runtime/src/navigator.rs|field_count|declaration|let field_count: u32 = 6;": 1, "crates/perry-runtime/src/node_stream_json.rs|keys_array|access|let keys = (*obj).keys_array;": 1, "crates/perry-runtime/src/node_stream_readwrite.rs|keys_array|access|let keys = (*obj).keys_array;": 1, @@ -161,8 +160,8 @@ "crates/perry-runtime/src/object/arguments.rs|field_count|access|std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32) as usize;": 2, "crates/perry-runtime/src/object/arguments.rs|keys_array|access|let keys = (*obj).keys_array;": 2, "crates/perry-runtime/src/object/class_registry/parent_static.rs|keys_array|access|let keys = (*obj).keys_array;": 1, - "crates/perry-runtime/src/object/class_registry/parent_static.rs|object_type|access|&& (*(ptr as *const ObjectHeader)).object_type == crate::error::OBJECT_TYPE_CLASS": 1, "crates/perry-runtime/src/object/class_registry/parent_static.rs|object_type|access|(*(obj as *mut ObjectHeader)).object_type = crate::error::OBJECT_TYPE_CLASS;": 1, + "crates/perry-runtime/src/object/class_registry/parent_static.rs|object_type|access|(*obj).object_type = crate::error::OBJECT_TYPE_REGULAR;": 1, "crates/perry-runtime/src/object/delete_rest.rs|field_count|access|assert_eq!(descriptor.live_inline_slot_count, (*obj).field_count);": 2, "crates/perry-runtime/src/object/delete_rest.rs|field_count|access|let field_count = (*obj).field_count;": 1, "crates/perry-runtime/src/object/delete_rest.rs|keys_array|access|(*obj).keys_array,": 1, @@ -187,23 +186,15 @@ "crates/perry-runtime/src/object/field_get_set/get_field_by_name_tail.rs|field_count|access|let _field_count = (*obj).field_count as usize;": 1, "crates/perry-runtime/src/object/field_get_set/get_field_by_name_tail.rs|field_count|access|std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32) as usize;": 1, "crates/perry-runtime/src/object/field_get_set/get_field_by_name_tail.rs|keys_array|access|let keys = (*obj).keys_array;": 1, - "crates/perry-runtime/src/object/field_get_set/get_field_by_name_tail.rs|object_type|access|if (*obj).object_type == crate::error::OBJECT_TYPE_CLASS && (*obj).class_id != 0 {": 1, - "crates/perry-runtime/src/object/field_get_set/get_field_by_name_tail.rs|object_type|access|let object_type = (*obj).object_type;": 1, "crates/perry-runtime/src/object/field_get_set/has_property.rs|keys_array|access|let keys = (*obj).keys_array;": 1, - "crates/perry-runtime/src/object/field_get_set/ic_miss.rs|field_count|access|std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32) as usize;": 1, - "crates/perry-runtime/src/object/field_get_set/ic_miss.rs|keys_array|access|(*obj).keys_array as u64": 1, - "crates/perry-runtime/src/object/field_get_set/ic_miss.rs|keys_array|access|let keys = (*obj).keys_array;": 2, - "crates/perry-runtime/src/object/field_get_set/ic_miss.rs|object_type|access|let is_regular = is_object && (*obj).object_type == crate::error::OBJECT_TYPE_REGULAR;": 1, + "crates/perry-runtime/src/object/field_get_set/ic_miss.rs|keys_array|access|let keys = (*obj).keys_array;": 1, "crates/perry-runtime/src/object/field_set_by_name.rs|field_count|access|(*o).field_count,": 1, "crates/perry-runtime/src/object/field_set_by_name.rs|field_count|access|if slot_idx >= (*o).field_count {": 1, "crates/perry-runtime/src/object/field_set_by_name.rs|keys_array|access|let keys = (*o).keys_array;": 1, - "crates/perry-runtime/src/object/field_set_by_name.rs|object_type|access|if (*o).object_type == crate::error::OBJECT_TYPE_REGULAR": 1, "crates/perry-runtime/src/object/field_set_by_name/fast_paths.rs|field_count|access|if idx >= (*obj).field_count {": 1, "crates/perry-runtime/src/object/field_set_by_name/fast_paths.rs|field_count|access|if slot_idx >= (*obj).field_count {": 1, "crates/perry-runtime/src/object/field_set_by_name/fast_paths.rs|field_count|access|std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32) as usize;": 2, "crates/perry-runtime/src/object/field_set_by_name/fast_paths.rs|keys_array|access|let keys = (*obj).keys_array;": 2, - "crates/perry-runtime/src/object/field_set_by_name/fast_paths.rs|object_type|access|if (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR": 1, - "crates/perry-runtime/src/object/field_set_by_name/fast_paths.rs|object_type|access||| (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR": 1, "crates/perry-runtime/src/object/field_set_by_name/tail.rs|field_count|access|if (*obj).field_count == 0 {": 1, "crates/perry-runtime/src/object/field_set_by_name/tail.rs|field_count|access|if new_index as u32 >= (*obj).field_count {": 2, "crates/perry-runtime/src/object/field_set_by_name/tail.rs|field_count|access|if slot_idx >= (*obj).field_count {": 1, @@ -211,25 +202,23 @@ "crates/perry-runtime/src/object/field_set_by_name/tail.rs|field_count|access|std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32) as usize;": 1, "crates/perry-runtime/src/object/field_set_by_name/tail.rs|keys_array|access|(*obj).keys_array,": 2, "crates/perry-runtime/src/object/field_set_by_name/tail.rs|keys_array|access|let keys = (*obj).keys_array;": 3, - "crates/perry-runtime/src/object/field_set_by_name/tail.rs|object_type|access|&& (*obj).object_type == crate::error::OBJECT_TYPE_REGULAR": 2, - "crates/perry-runtime/src/object/field_set_by_name/tail.rs|object_type|access|let object_type = (*obj).object_type;": 1, - "crates/perry-runtime/src/object/field_set_by_name/write_helpers.rs|object_type|access|if (*obj).object_type != crate::error::OBJECT_TYPE_CLASS": 1, + "crates/perry-runtime/src/object/gc_slots.rs|field_count|access|.unwrap_or((*obj).field_count as usize);": 1, + "crates/perry-runtime/src/object/gc_slots.rs|keys_array|access|(*obj).keys_array = descriptor.keys as usize as *mut ArrayHeader;": 1, + "crates/perry-runtime/src/object/gc_slots.rs|keys_array|access|Some(&mut (*obj).keys_array as *mut _ as *mut u64)": 1, + "crates/perry-runtime/src/object/gc_slots.rs|keys_array|access|if (*obj).keys_array.is_null() {": 1, "crates/perry-runtime/src/object/map_set_subclass.rs|field_count|access|assert_eq!(unsafe { (*obj).field_count }, 3);": 1, "crates/perry-runtime/src/object/map_set_subclass.rs|object_type|access|assert_eq!(unsafe { (*obj).object_type }, OBJECT_TYPE_REGULAR);": 5, "crates/perry-runtime/src/object/mod.rs|field_count|access|(*obj).field_count = field_count;": 1, "crates/perry-runtime/src/object/mod.rs|field_count|access|if (*obj).field_count != field_count {": 1, - "crates/perry-runtime/src/object/mod.rs|field_count|access|let field_count = (*obj).field_count as usize;": 1, "crates/perry-runtime/src/object/mod.rs|field_count|declaration|field_count: 0,": 1, "crates/perry-runtime/src/object/mod.rs|field_count|declaration|field_count: u32,": 1, "crates/perry-runtime/src/object/mod.rs|field_count|declaration|pub field_count: u32,": 1, "crates/perry-runtime/src/object/mod.rs|field_count|declaration|pub(super) unsafe fn set_object_live_slot_count(obj: *mut ObjectHeader, field_count: u32) {": 1, "crates/perry-runtime/src/object/mod.rs|keys_array|access|&(*obj).keys_array as *const _ as usize,": 1, "crates/perry-runtime/src/object/mod.rs|keys_array|access|(*obj).keys_array = keys_array;": 1, - "crates/perry-runtime/src/object/mod.rs|keys_array|access|Some(&mut (*obj).keys_array as *mut _ as *mut u64)": 1, "crates/perry-runtime/src/object/mod.rs|keys_array|access|crate::gc::runtime_store_root_raw_mut_ptr_slot(&mut entry.keys_array, keys_array);": 2, - "crates/perry-runtime/src/object/mod.rs|keys_array|access|if (*obj).keys_array != keys_array {": 2, - "crates/perry-runtime/src/object/mod.rs|keys_array|access|if obj.is_null() || (*obj).keys_array.is_null() {": 1, "crates/perry-runtime/src/object/mod.rs|keys_array|access|let inline = unsafe { (*st.object_hot.shape_inline_cache.get())[slot].keys_array as usize };": 1, + "crates/perry-runtime/src/object/mod.rs|keys_array|access|let keys_changed = (*obj).keys_array != keys_array;": 1, "crates/perry-runtime/src/object/mod.rs|keys_array|access|return (entry.keys_array, entry.runtime_shape_id);": 1, "crates/perry-runtime/src/object/mod.rs|keys_array|access|visitor.visit_raw_mut_ptr_slot(&mut entry.keys_array);": 1, "crates/perry-runtime/src/object/mod.rs|keys_array|declaration|fn shape_cache_insert(shape_id: u32, keys_array: *mut ArrayHeader) {": 1, @@ -244,9 +233,6 @@ "crates/perry-runtime/src/object/mod.rs|object_type|declaration|object_type: u32,": 1, "crates/perry-runtime/src/object/mod.rs|object_type|declaration|pub object_type: u32,": 1, "crates/perry-runtime/src/object/namespace_create.rs|keys_array|access|(*obj).keys_array = 0x2800_0203usize as *mut _;": 1, - "crates/perry-runtime/src/object/native_call_method.rs|keys_array|access|let keys = (*obj).keys_array;": 2, - "crates/perry-runtime/src/object/native_call_method.rs|object_type|access|if (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR {": 1, - "crates/perry-runtime/src/object/native_call_method.rs|object_type|access|let object_type = (*obj).object_type;": 1, "crates/perry-runtime/src/object/native_call_method/collection_methods.rs|keys_array|access|let keys = (*obj).keys_array;": 1, "crates/perry-runtime/src/object/native_call_method/handle_methods.rs|keys_array|access|let keys = (*obj).keys_array;": 1, "crates/perry-runtime/src/object/object_ops.rs|keys_array|declaration|pub(crate) use keys_array::{": 1, @@ -266,8 +252,10 @@ "crates/perry-runtime/src/object/object_ops_frozen.rs|keys_array|access|let keys = (*obj).keys_array;": 1, "crates/perry-runtime/src/object/reflect_support.rs|keys_array|access|let keys_handle = scope.root_raw_mut_ptr((*obj).keys_array);": 1, "crates/perry-runtime/src/object/shapes.rs|field_count|access|&& d.live_inline_slot_count == (*obj).field_count": 1, + "crates/perry-runtime/src/object/shapes.rs|field_count|access|(*obj).field_count,": 1, "crates/perry-runtime/src/object/shapes.rs|field_count|access|assert_eq!(descriptor.live_inline_slot_count, (*obj).field_count);": 1, - "crates/perry-runtime/src/object/shapes.rs|field_count|access|let Ok(id) = shape_descriptor_ensure(keys, key_count, (*obj).field_count) else {": 2, + "crates/perry-runtime/src/object/shapes.rs|field_count|access|let id = shape_descriptor_ensure(keys, key_count, (*obj).field_count)": 1, + "crates/perry-runtime/src/object/shapes.rs|field_count|access||| install_external_shape_id(runtime_shape_id, keys, key_count, (*obj).field_count);": 1, "crates/perry-runtime/src/object/shapes.rs|field_count|declaration|field_count: 2,": 1, "crates/perry-runtime/src/object/shapes.rs|keys_array|access|(*a).keys_array,": 1, "crates/perry-runtime/src/object/shapes.rs|keys_array|access|(*b).keys_array,": 1, @@ -290,40 +278,22 @@ "crates/perry-runtime/src/perf_hooks.rs|keys_array|access|recorded != 0 && (*obj).keys_array as usize == recorded": 1, "crates/perry-runtime/src/pointer_event.rs|field_count|declaration|let field_count: u32 = 4;": 1, "crates/perry-runtime/src/promise/then_probe.rs|keys_array|access|let keys = (*obj).keys_array;": 2, - "crates/perry-runtime/src/promise/then_probe.rs|object_type|access|if (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR {": 1, "crates/perry-runtime/src/proxy.rs|object_type|access|&& (*(addr as *const crate::ObjectHeader)).object_type": 1, - "crates/perry-runtime/src/proxy/put_value.rs|field_count|access|(*first).field_count,": 1, - "crates/perry-runtime/src/proxy/put_value.rs|field_count|access|let alloc_limit = std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32);": 1, "crates/perry-runtime/src/proxy/put_value.rs|field_count|access|object_array_numeric_write_slots(array, &keys[..field_count as usize], receiver_count)": 1, "crates/perry-runtime/src/proxy/put_value.rs|field_count|access|slots[..field_count as usize]": 1, - "crates/perry-runtime/src/proxy/put_value.rs|field_count|access|std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32) as usize;": 1, - "crates/perry-runtime/src/proxy/put_value.rs|field_count|access|std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32);": 1, - "crates/perry-runtime/src/proxy/put_value.rs|field_count|access|unsafe { std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32) };": 1, "crates/perry-runtime/src/proxy/put_value.rs|field_count|declaration|field_count: u32,": 1, - "crates/perry-runtime/src/proxy/put_value.rs|keys_array|access|(*obj).keys_array as u64": 1, - "crates/perry-runtime/src/proxy/put_value.rs|keys_array|access|let keys = (*obj).keys_array;": 3, - "crates/perry-runtime/src/proxy/put_value.rs|object_type|access|if (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR": 4, "crates/perry-runtime/src/safe_area.rs|field_count|declaration|let field_count: u32 = 4;": 1, "crates/perry-runtime/src/thread.rs|field_count|access|for i in 0..field_count {": 1, "crates/perry-runtime/src/thread.rs|field_count|access|let field_count = (*obj).field_count as usize;": 1, "crates/perry-runtime/src/thread.rs|keys_array|access|let keys = if !(*obj).keys_array.is_null() {": 1, "crates/perry-runtime/src/thread.rs|keys_array|access|let keys_arr = (*obj).keys_array;": 1, - "crates/perry-runtime/src/typed_feedback.rs|field_count|access|std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32);": 1, - "crates/perry-runtime/src/typed_feedback.rs|keys_array|access|(*ptr).keys_array as usize": 2, - "crates/perry-runtime/src/typed_feedback.rs|keys_array|access|let keys = (*obj).keys_array;": 1, - "crates/perry-runtime/src/typed_feedback.rs|keys_array|access|let keys = (*ptr).keys_array;": 1, - "crates/perry-runtime/src/typed_feedback/guards.rs|field_count|access|&& expected_field_index < (*obj).field_count": 2, - "crates/perry-runtime/src/typed_feedback/guards.rs|field_count|access|&& expected_field_index < (*obj).field_count;": 1, - "crates/perry-runtime/src/typed_feedback/guards.rs|keys_array|access|&& std::ptr::eq((*obj).keys_array as *const ArrayHeader, expected_keys)": 1, - "crates/perry-runtime/src/typed_feedback/guards.rs|keys_array|access|*out_keys = (*obj).keys_array as u64;": 1, - "crates/perry-runtime/src/typed_feedback/guards.rs|keys_array|access|let keys = (*obj).keys_array;": 1, - "crates/perry-runtime/src/typed_feedback/guards.rs|keys_array|access|let shape_addr = (*obj).keys_array as usize;": 2, - "crates/perry-runtime/src/typed_feedback/guards.rs|keys_array|access|return ((*obj).keys_array as usize, (*obj).class_id, gc_type, false);": 1, - "crates/perry-runtime/src/typed_feedback/guards.rs|keys_array|access||| !std::ptr::eq((*obj).keys_array, expected_keys)": 1, - "crates/perry-runtime/src/typed_feedback/guards.rs|object_type|access|if (*obj).object_type != crate::error::OBJECT_TYPE_REGULAR {": 2, - "crates/perry-runtime/src/typed_feedback/guards.rs|object_type|access|let shape_ok = (*obj).object_type == crate::error::OBJECT_TYPE_REGULAR": 1, - "crates/perry-runtime/src/typed_feedback/guards.rs|object_type|access|let valid = (*obj).object_type == crate::error::OBJECT_TYPE_REGULAR": 1, - "crates/perry-runtime/src/typed_feedback/tests.rs|keys_array|access|assert_ne!(unsafe { (*obj).keys_array }, expected_keys);": 1, + "crates/perry-runtime/src/typed_feedback/tests.rs|field_count|access|(*obj).field_count = 0;": 1, + "crates/perry-runtime/src/typed_feedback/tests.rs|field_count|access|(*obj).field_count = original_field_count;": 1, + "crates/perry-runtime/src/typed_feedback/tests.rs|field_count|access|let original_field_count = unsafe { (*obj).field_count };": 1, + "crates/perry-runtime/src/typed_feedback/tests.rs|keys_array|access|(*obj).keys_array = original_keys;": 1, + "crates/perry-runtime/src/typed_feedback/tests.rs|keys_array|access|(*obj).keys_array = std::ptr::null_mut();": 1, + "crates/perry-runtime/src/typed_feedback/tests.rs|keys_array|access|&(*obj).keys_array as *const _ as usize,": 1, + "crates/perry-runtime/src/typed_feedback/tests.rs|keys_array|access|assert_ne!(unsafe { (*obj).keys_array }, original_keys);": 1, "crates/perry-runtime/src/typed_feedback/tests.rs|keys_array|access|let keys = unsafe { (*obj).keys_array };": 1, "crates/perry-runtime/src/url/search_params.rs|keys_array|access|let keys_arr = (*obj).keys_array;": 2, "crates/perry-runtime/src/url/search_params.rs|keys_array|access|let keys_arr = (*params).keys_array;": 1, @@ -358,11 +328,11 @@ }, "summary": { "codegen_object_header_size_sites": 32, - "raw_member_files": 104, + "raw_member_files": 99, "raw_member_sites": { - "field_count": 165, - "keys_array": 195, - "object_type": 54 + "field_count": 159, + "keys_array": 181, + "object_type": 31 } } } diff --git a/tests/fixtures/issue_8075_provider_gc/host.rs b/tests/fixtures/issue_8075_provider_gc/host.rs index 054693fe07..d82efb3ca9 100644 --- a/tests/fixtures/issue_8075_provider_gc/host.rs +++ b/tests/fixtures/issue_8075_provider_gc/host.rs @@ -134,7 +134,7 @@ fn validate_frame(api: RuntimeApi, handler: Handler, invocation: usize) -> Resul let frame = unsafe { std::slice::from_raw_parts(data, length) }; if frame.len() != 15 + EXPECTED_BODY.len() { return Err(format!( - "invocation {invocation}: frame length {}, expected {}", + "invocation {invocation}: frame length {}, expected {}; frame={frame:?}", frame.len(), 15 + EXPECTED_BODY.len() ));