diff --git a/benchmarks/json_polyglot/run.sh b/benchmarks/json_polyglot/run.sh index 96f57b4354..b0a9a12e76 100755 --- a/benchmarks/json_polyglot/run.sh +++ b/benchmarks/json_polyglot/run.sh @@ -542,8 +542,83 @@ echo echo "Wrote $(pwd)/RESULTS.md" cat RESULTS.md +# --------------------------------------------------------------------------- +# Cross-runtime correctness gate (#7264). +# +# Every workload prints `checksum:` — a fold over the parsed data plus the +# length of the re-serialized blob. Perry, node and bun must agree exactly; a +# disagreement means one of them produced different DATA, which is a +# correctness bug, not a measurement artifact. +# +# This used to be checked only inside the optional JSON publisher below, so a +# mismatch (a) did not fail a plain `run.sh` at all, and (b) under +# run_public_baseline.sh surfaced ~40 minutes later as a confusing "could not +# load json-polyglot.json" at final assembly. Check it here, immediately after +# the runs, unconditionally, and name the offending runtimes. +# --------------------------------------------------------------------------- +checksum_failed=0 +for workload in roundtrip field_access; do + # Mirror the publisher's cell selection: perry's optimized row, node/bun's + # idiomatic row. Emits one " " line per runtime found. + observed=$(awk -F'\t' -v w="$workload" ' + $1 != w { next } + { + split($3, parts, " ") + runtime = parts[1] + want = (runtime == "perry") ? "optimized" : "idiomatic" + if (runtime != "perry" && runtime != "node" && runtime != "bun") next + if ($2 != want || runtime in seen) next + split($5, cs, ",") + if (cs[1] == "") next + seen[runtime] = cs[1] + order[++n] = runtime + } + END { for (i = 1; i <= n; i++) print order[i], seen[order[i]] } + ' "$RAW_RESULTS_FILE") + # A publishing run REQUIRES all three: the publisher indexes + # selected[(workload, runtime)] for perry, node and bun unconditionally, so + # a missing row is a hard error there — and "they all agree" is vacuous when + # only one runtime reported. A local run without node/bun installed is a + # legitimate use of this script, so only enforce presence when publishing. + if [[ -n "$PUBLIC_JSON_OUT" ]]; then + for required in perry node bun; do + if ! grep -q "^$required " <<< "$observed"; then + checksum_failed=1 + { + echo + echo "ERROR: json_polyglot/$workload — no checksum row for '$required'." + echo " A published artifact compares perry against BOTH node and bun;" + echo " with a runtime missing there is nothing to compare against." + echo " Install it (or drop \$PUBLIC_BENCH_JSON_OUT for a local run)." + } >&2 + fi + done + fi + [[ -z "$observed" ]] && continue + distinct=$(awk '{ print $2 }' <<< "$observed" | sort -u | wc -l | tr -d ' ') + if [[ "$distinct" -ne 1 ]]; then + checksum_failed=1 + { + echo + echo "ERROR: json_polyglot/$workload — runtimes disagree on the DATA." + echo " Each line is ; they must all be equal." + sed 's/^/ /' <<< "$observed" + echo " This is a correctness bug in whichever runtime is the odd one out," + echo " not benchmark noise. The results above are not publishable." + } >&2 + fi +done +if [[ "$checksum_failed" -ne 0 ]]; then + echo >&2 + echo "json_polyglot: FAILED the cross-runtime checksum gate — not publishing." >&2 + exit 1 +fi + if [[ -n "$PUBLIC_JSON_OUT" ]]; then mkdir -p "$(dirname "$PUBLIC_JSON_OUT")" + # Never leave a stale artifact behind: a previous run's file must not be + # mistaken for this run's output if the publisher aborts. + rm -f "$PUBLIC_JSON_OUT" PYTHONPATH="$(cd "$PERRY_ROOT" && pwd)" python3 - "$RAW_RESULTS_FILE" "$PUBLIC_JSON_OUT" "$(cd "$PERRY_ROOT" && pwd)" "$RUNS" "$PIN_NOTE" <<'PY' import json, shutil, subprocess, sys from datetime import datetime, timezone @@ -614,5 +689,19 @@ component = { } Path(output_path).write_text(json.dumps(component, indent=2) + "\n") PY + # The script runs under `set -uo pipefail`, NOT `-e`: without this check a + # publisher that aborted via `raise SystemExit(...)` still fell through to + # the success line below and run.sh exited 0 having written nothing (#7264). + publish_status=$? + if [[ "$publish_status" -ne 0 || ! -s "$PUBLIC_JSON_OUT" ]]; then + { + echo + echo "ERROR: json_polyglot did not publish machine-readable results." + echo " The publisher exited $publish_status and $PUBLIC_JSON_OUT is missing or empty." + echo " The message above this line is the reason; nothing downstream can use this leg." + } >&2 + [[ "$publish_status" -eq 0 ]] && publish_status=1 + exit "$publish_status" + fi echo "Machine-readable results: $PUBLIC_JSON_OUT" fi diff --git a/changelog.d/7265-json-array-element-overflow-fields.md b/changelog.d/7265-json-array-element-overflow-fields.md new file mode 100644 index 0000000000..11b4cd1c74 --- /dev/null +++ b/changelog.d/7265-json-array-element-overflow-fields.md @@ -0,0 +1,17 @@ +fix(json): `JSON.stringify` silently dropped every array element's properties past the inline-slot floor (#7264). + +`JSON.stringify(arr)` truncated **every** element of a homogeneous array of objects to 4 properties — no error, no warning, just short output — as soon as the array came from `JSON.parse` and any property of any element had been read. `JSON.stringify(JSON.parse(x))` is a routine idiom, so anything that parses JSON, reads a field, and re-serialises (a proxy, a cache layer, a request handler) could emit truncated records. + +**Root cause.** The array fast path builds one shape template from element 0 and reuses its pre-formatted key prefixes for every element. It sized that template from `min(keys_len, field_count)`. `keys_len` is the *logical* property count; `field_count` is *physical* — it never exceeds the object's inline slot allocation. An object grown by name past `INLINE_SLOT_FLOOR` keeps `field_count` pinned at the floor and parks the remaining values in overflow storage. `JSON.parse`'s lazy-tape materializer builds exactly that shape (`js_object_alloc(0, 0)` plus one `js_object_set_field_by_name` per key), so a 6-key record reports `field_count == 4` and the template emitted — and read — only 4 slots. + +Only the *array* path was wrong, which is what made this so hard to see: the single-object path already knew the invariant (its `has_overflow_fields` guard bails out of the template when `keys_len > field_count`), so `JSON.stringify(parsed[0])`, `parsed.map(o => JSON.stringify(o))` and `Object.keys(parsed[0])` all reported the truth while the whole-array result did not. + +`build_shape_prefix_template` now sizes the template from `keys_len` and routes slot reads through `template_field_bits` / `field_bits_at`, which read inline below `max(field_count, INLINE_SLOT_FLOOR)` and fall through to `js_object_get_field`'s overflow lookup above it — the same rule `stringify_object_inner` has used since #307. The `min` was never needed for the opposite skew either: a pre-sized object (`js_object_alloc(0, 8)` holding 2 keys) has `field_count > keys_len`, and `keys_len` already stops at the last real key. + +Latent since the shape template landed (v0.5.65), where it only bit objects with 9+ properties; exposed for ordinary 5–8-field records by 6958a5a8d (#6712), which lowered `INLINE_SLOT_FLOOR` from 8 to 4. That is why the `2026-07-13` public baseline passed the same checksum gate this failed. + +**Second defect, same emitter.** The template's primitive-only fast path is chosen by *sampling element 0*, then trusted every later element to be primitive too. A function- or symbol-valued property in a later element must be omitted per `SerializeJSONObject`, but the fast path had already written the key prefix and rendered the closure as `null` — emitting a member that must not exist (`{"f2":null}` where node emits nothing). Only the general path pre-scanned for it. It now rolls the buffer back and defers to the slow path, exactly as it already did for a stray `undefined`. Applies to inline slots as much as overflow ones. + +**Harness.** `benchmarks/json_polyglot/run.sh` printed `Machine-readable results: ` and exited 0 while its embedded publisher had already aborted via `raise SystemExit(...)` and written nothing — the script runs under `set -uo pipefail`, not `-e`. `run_public_baseline.sh` therefore sailed past it and failed ~40 minutes later with a misleading "could not load json-polyglot.json". Two changes: a cross-runtime checksum gate now runs unconditionally right after the benchmark cells, names the disagreeing runtimes, and fails the leg immediately; and the publisher's exit status plus the existence of a non-empty output file are both checked before the success line prints. A stale artifact from a previous run is also removed up front so it can't be mistaken for this run's output. + +Verified: `benchmarks/json_polyglot/bench_field_access.ts` now checksums `2552985550` and `bench.ts` `53735550` under perry, node 26.5.1 and bun 1.3.14 alike (perry previously gave `2538318800` on the first), unblocking the public-baseline regeneration in #7257. Field counts 1–12 round-trip byte-identically; the untouched-array, one-read and loop-read forms all agree; the nested-subtree shape from the report is byte-identical. New `--lib` unit tests in `crates/perry-runtime/src/json/mod.rs` (which run per-PR, unlike `crates/*/tests/*.rs`) plus `test-files/test_gap_json_array_element_overflow_fields.ts`. Full 471-test gap suite and `cargo test -p perry-runtime --lib` show no regressions. diff --git a/crates/perry-runtime/src/json/mod.rs b/crates/perry-runtime/src/json/mod.rs index 66cecfd598..c8cc57a867 100644 --- a/crates/perry-runtime/src/json/mod.rs +++ b/crates/perry-runtime/src/json/mod.rs @@ -10,6 +10,7 @@ //! - `parser` — `DirectParser` recursive-descent implementation //! - `parse_api` — `js_json_parse` / `js_json_parse_typed_array` FFI //! - `stringify` — core stringify traversal (object/array/scalar emitters) +//! - `stringify_shape_template` — homogeneous-array shape templates //! - `stringify_api` — `js_json_stringify*` / `js_json_get_*` FFI entries //! - `replacer` — replacer + pretty/indent + array-replacer variants //! - `reviver` — `JSON.parse(text, reviver)` support @@ -33,6 +34,7 @@ mod stringify; mod stringify_api; mod stringify_buffer; mod stringify_scalars; +mod stringify_shape_template; mod stringify_tojson_probe; // Public FFI re-exports — preserve the `crate::json::js_json_*` path used by @@ -63,12 +65,12 @@ pub(crate) use simd::find_string_terminator; pub(crate) use stringify::{ arm_to_json_result_guard, estimate_json_size, is_closure_value, is_object_pointer, is_symbol_value, object_get_to_json, stringify_value, write_escaped_string, write_number, - ShapeTemplate, }; pub(crate) use stringify_api::{redirect_lazy_to_materialized, try_stringify_lazy_array}; pub(crate) use stringify_buffer::{ stringify_buffer, stringify_buffer_pretty, stringify_typed_array, stringify_typed_array_pretty, }; +pub(crate) use stringify_shape_template::ShapeTemplate; pub(crate) use stringify_tojson_probe::{ current_to_json_key_arg, invalidate_object_proto_tojson_state, reset_to_json_key, set_to_json_key_index, set_to_json_key_str, set_to_json_key_value, to_json_definitely_absent, @@ -935,4 +937,158 @@ mod tests { "\"a\\bb\\fc\\u000bd\"" ); } + + /// Materialize `src` exactly the way `JSON.parse` does once a property + /// read forces the lazy tape to become real objects, and return both the + /// value and the resulting `(field_count, keys_len)` of element 0. + unsafe fn materialized_array(src: &[u8]) -> (JSValue, u32, u32) { + let tape = crate::json_tape::build_tape(src).expect("tape builds for valid JSON"); + let value = crate::json_tape::materialize(&tape, src); + let arr = (value.bits() & POINTER_MASK) as *mut crate::ArrayHeader; + let elem0 = crate::array::js_array_get(arr, 0); + let obj = (elem0.bits() & POINTER_MASK) as *const crate::ObjectHeader; + (value, (*obj).field_count, (*(*obj).keys_array).length) + } + + #[test] + fn stringify_array_keeps_every_property_of_name_grown_elements() { + // #7264: `JSON.stringify` silently dropped every property past the + // 4th from EVERY element of a homogeneous array. + // + // The array fast path built its shape template with + // `min(keys_len, field_count)`. `field_count` is PHYSICAL — it never + // exceeds the object's inline slot allocation, so an object grown by + // name past `INLINE_SLOT_FLOOR` reports the floor (4) while the + // remaining values live in overflow storage. `JSON.parse`'s tape + // materializer produces exactly that shape (`js_object_alloc(0, 0)` + + // `js_object_set_field_by_name` per key), so `JSON.stringify` of a + // parsed-and-touched array truncated every record to 4 fields. + // + // Latent since the shape template landed (v0.5.65, 9-field threshold); + // exposed for ordinary 5–8-field records by #6712 (floor 8 → 4). + let src = br#"[{"f0":0,"f1":1,"f2":2,"f3":3,"f4":4,"f5":5},{"f0":10,"f1":11,"f2":12,"f3":13,"f4":14,"f5":15}]"#; + unsafe { + let (value, field_count, keys_len) = materialized_array(src); + // Assert the gate's SUBJECT is live: this test is only meaningful + // while the materializer really does leave `field_count` below + // `keys_len` (the overflow shape). If a future change pre-sizes + // the allocation, this assertion fires and the test must be + // re-pointed rather than silently passing on the easy path. + assert!( + field_count < keys_len, + "precondition lost: tape-materialized element is no longer an \ + overflow shape (field_count={field_count}, keys_len={keys_len}) — \ + re-point this regression test at a shape that still is" + ); + assert_eq!(keys_len, 6); + + let output = js_json_stringify(f64::from_bits(value.bits()), TYPE_ARRAY); + assert_eq!( + str_from_header(output).unwrap(), + std::str::from_utf8(src).unwrap(), + "array-element serialization must emit all {keys_len} properties" + ); + } + } + + #[test] + fn stringify_array_round_trips_every_field_count_across_the_inline_floor() { + // The truncation was invisible at ≤ INLINE_SLOT_FLOOR fields and + // produced IDENTICAL output for 5, 6 and 8 — the tell that emission + // stopped at the floor. Sweep across the boundary. + for n in 1..=10usize { + let record = |base: usize| { + let fields: Vec = + (0..n).map(|f| format!("\"f{f}\":{}", base + f)).collect(); + format!("{{{}}}", fields.join(",")) + }; + let src = format!("[{},{}]", record(0), record(100)); + unsafe { + let tape = crate::json_tape::build_tape(src.as_bytes()).expect("tape"); + let value = crate::json_tape::materialize(&tape, src.as_bytes()); + let output = js_json_stringify(f64::from_bits(value.bits()), TYPE_ARRAY); + assert_eq!( + str_from_header(output).unwrap(), + src.as_str(), + "{n}-field records must round-trip" + ); + } + } + } + + #[test] + fn stringify_array_keeps_nested_objects_and_arrays_past_the_inline_floor() { + // The shape reported in #7264: the 5th property is a nested object and + // the 4th an array, so the dropped field was a whole subtree. + let src = br#"[{"id":0,"name":"item_0","value":0,"tags":["tag_0","tag_0"],"nested":{"x":0,"y":0}},{"id":1,"name":"item_1","value":3,"tags":["tag_1","tag_1"],"nested":{"x":1,"y":2}}]"#; + unsafe { + let (value, field_count, keys_len) = materialized_array(src); + assert!(field_count < keys_len, "precondition: overflow shape"); + let output = js_json_stringify(f64::from_bits(value.bits()), TYPE_ARRAY); + assert_eq!( + str_from_header(output).unwrap(), + std::str::from_utf8(src).unwrap() + ); + } + } + + #[test] + fn stringify_array_omits_a_function_valued_property_of_a_later_element() { + // Second defect found in the same emitter while fixing #7264. The + // template's primitive-only fast path is chosen by SAMPLING element 0; + // it then trusted every later element to be primitive too. A function + // (or symbol) value in a later element must be OMITTED per + // SerializeJSONObject, but the fast path had already written the key + // prefix and rendered the closure as `null` — emitting a member that + // must not exist. Only the general (non-primitive-only) path pre-scanned + // for it. Applies to inline slots as much as overflow ones. + let src = br#"[{"f0":0,"f1":1,"f2":2,"f3":3,"f4":4,"f5":5},{"f0":10,"f1":11,"f2":12,"f3":13,"f4":14,"f5":15}]"#; + unsafe { + let (value, _, _) = materialized_array(src); + let arr = (value.bits() & POINTER_MASK) as *mut crate::ArrayHeader; + let elem1 = crate::array::js_array_get(arr, 1); + let obj1 = (elem1.bits() & POINTER_MASK) as *mut crate::ObjectHeader; + // `f4` is at index 4 — an OVERFLOW slot on this shape, so this also + // covers the fallback plumbing added for overflow reads. + let closure = crate::closure::js_closure_alloc(std::ptr::null(), 0); + let key = js_string_from_bytes(b"f4".as_ptr(), 2); + crate::object::js_object_set_field_by_name( + obj1, + key, + f64::from_bits(POINTER_TAG | (closure as u64 & POINTER_MASK)), + ); + + let output = js_json_stringify(f64::from_bits(value.bits()), TYPE_ARRAY); + assert_eq!( + str_from_header(output).unwrap(), + r#"[{"f0":0,"f1":1,"f2":2,"f3":3,"f4":4,"f5":5},{"f0":10,"f1":11,"f2":12,"f3":13,"f5":15}]"# + ); + } + } + + #[test] + fn stringify_array_stops_at_the_last_real_key_when_slots_were_pre_sized() { + // The opposite skew, and the reason the truncating `min` existed: an + // object allocated with MORE inline slots than it has keys + // (`js_object_alloc(0, 8)` holding 2 properties). `keys_len` alone must + // still stop at the last real key rather than dumping padding slots. + unsafe { + let mut arr = crate::array::js_array_alloc(2); + for base in [1.0f64, 10.0f64] { + let obj = crate::object::js_object_alloc(0, 8); + for (i, name) in ["a", "b"].iter().enumerate() { + let key = js_string_from_bytes(name.as_ptr(), name.len() as u32); + crate::object::js_object_set_field_by_name(obj, key, base + i as f64); + } + assert!((*obj).field_count >= (*(*obj).keys_array).length); + arr = crate::array::js_array_push(arr, JSValue::object_ptr(obj as *mut u8)); + } + let boxed = crate::value::js_nanbox_pointer(arr as i64); + let output = js_json_stringify(boxed, TYPE_ARRAY); + assert_eq!( + str_from_header(output).unwrap(), + r#"[{"a":1,"b":2},{"a":10,"b":11}]"# + ); + } + } } diff --git a/crates/perry-runtime/src/json/stringify.rs b/crates/perry-runtime/src/json/stringify.rs index 538fae29e7..9267e08d3d 100644 --- a/crates/perry-runtime/src/json/stringify.rs +++ b/crates/perry-runtime/src/json/stringify.rs @@ -13,6 +13,11 @@ pub(crate) use super::stringify_scalars::{ bigint_apply_to_json, serialize_bigint, throw_bigint_serialize, write_escaped_string, write_number, }; +// The homogeneous-array shape template lives in a sibling (file-size gate); +// both the object and the array emitter below drive it. +use super::stringify_shape_template::{ + build_shape_prefix_template, shape_template_for, try_emit_shape_element, +}; // ─── JSON.stringify ─────────────────────────────────────────────────────────── @@ -1329,355 +1334,6 @@ pub(crate) unsafe fn stringify_array(ptr: *const u8, buf: &mut String) { stringify_array_depth(ptr, buf, 0) } -/// Cached shape template for a homogeneous array of objects. -pub(crate) struct ShapeTemplate { - pub(crate) keys_arr: *mut crate::ArrayHeader, - pub(crate) prefixes: Vec, - pub(crate) shape_fields: u32, - /// True when element 0's fields are all primitives (no POINTER_TAG / - /// UNDEFINED). Lets the emit path skip its per-element pre-scan. - pub(crate) primitive_only: bool, -} - -/// Look up (or build & insert) the shape template for an object. Returns -/// `None` if the object isn't templatable (no keys array, too many fields, -/// malformed key strings) or if the cache is full and missed. -/// -/// Returns a raw pointer because lifetimes can't survive the TLS borrow. -/// The pointer stays valid until the next `take_shape_cache` (top-level -/// entry/exit) — within one stringify traversal we only `push`, and -/// `Box`'s heap address is stable across `Vec` growth. -#[inline] -pub(crate) unsafe fn shape_template_for(obj_ptr: *const u8) -> Option<*const ShapeTemplate> { - let obj = obj_ptr as *const crate::ObjectHeader; - let keys_arr = (*obj).keys_array; - if keys_arr.is_null() { - return None; - } - - SHAPE_CACHE.with(|c| { - // Fast path: linear scan from the back — recently-used entries - // cluster there for typical traversal orders (shape A's elements - // recurse into shape B repeatedly). - { - let cache = c.borrow(); - for entry in cache.iter().rev() { - if entry.0 == keys_arr { - return Some(&*entry.1 as *const ShapeTemplate); - } - } - if cache.len() >= SHAPE_CACHE_CAP { - return None; - } - } - - // Miss — build, insert, return raw pointer to the boxed template. - let elem_bits = make_pointer_bits(obj_ptr); - let template = build_shape_prefix_template(elem_bits)?; - let mut cache = c.borrow_mut(); - // Re-check cap after the borrow round-trip (a recursive call - // during template build could have filled the cache). - if cache.len() >= SHAPE_CACHE_CAP { - return None; - } - cache.push((keys_arr, Box::new(template))); - Some(&*cache.last().unwrap().1 as *const ShapeTemplate) - }) -} - -/// Build a per-shape key-prefix template for a homogeneous array of objects. -/// -/// When every element of an array shares the same `keys_array` pointer (same -/// shape), we can pre-format the key portion of each field once and reuse it -/// across every element — turning the per-field key lookup (load key f64, -/// extract pointer, `str_from_header`, 3 `push`/`push_str` calls) into a -/// single `push_str` of a cached prefix. -/// -/// Prefix layout for N fields with keys k0..kN-1: -/// `prefixes[0] = "{\"k0\":"` (opening brace fused with first key) -/// `prefixes[f>0] = ",\"kf\":"` (comma fused with key) -/// Close with `}`. This compresses ~7 per-field write ops down to ~2. -/// -/// Returns `None` when the first element isn't a regular object, the keys -/// array is invalid, or any key string is malformed — callers fall back to -/// the generic slow path in that case. -pub(crate) unsafe fn build_shape_prefix_template(first_elem_bits: u64) -> Option { - let tag = first_elem_bits & 0xFFFF_0000_0000_0000; - let first_ptr = if tag == POINTER_TAG { - (first_elem_bits & POINTER_MASK) as *const u8 - } else if is_raw_pointer(first_elem_bits) { - first_elem_bits as *const u8 - } else { - return None; - }; - // Issue #639: Buffer / Uint8Array have no GcHeader, so `gc_obj_type` - // would read 8 bytes before the BufferHeader (unrelated memory) and - // could randomly return GC_TYPE_OBJECT. Bail to the per-element - // slow path which dispatches via `is_registered_buffer`. - if crate::buffer::is_registered_buffer(first_ptr as usize) { - return None; - } - if gc_obj_type(first_ptr) != crate::gc::GC_TYPE_OBJECT { - return None; - } - let obj = first_ptr as *const crate::ObjectHeader; - // A non-zero `class_id` means this object may resolve `toJSON` (or other - // serialization-affecting methods) on its prototype / class vtable — which - // the prefix-template emit path can't see (it only inspects own fields). - // Bail to the per-element slow path (`stringify_object_inner`), which - // probes the prototype chain via `object_get_to_json`. Plain data object - // literals and `JSON.parse` output carry `class_id == 0`, so the - // array-of-objects fast path is unaffected for them. (#321 — a homogeneous - // array of `class { toJSON() {…} }` instances must honour the prototype - // `toJSON`.) - if (*obj).class_id != 0 { - return None; - } - // #6519: a URL instance is a class_id-0 object but must serialize as its - // `href` string (handled by `stringify_object_inner`'s URL branch), never - // as a templated dump of its 12 internal fields (which would also walk the - // `searchParams` back-reference and throw). Bail so an array whose first - // element is a URL routes every element through the per-element slow path. - if crate::url::is_url_object_shape(obj as *mut crate::ObjectHeader) { - return None; - } - let keys_arr = (*obj).keys_array; - if keys_arr.is_null() { - return None; - } - // #2438: array-index keys must enumerate first in ascending numeric order, - // which the insertion-ordered prefix template can't express. Bail to the - // generic slow path (`stringify_object_inner`), which reorders per spec. - if crate::object::keys_contain_array_index(keys_arr) { - return None; - } - let keys_len = (*keys_arr).length; - let field_count = (*obj).field_count; - let shape_fields = std::cmp::min(keys_len, field_count); - if shape_fields == 0 || shape_fields > 32 { - return None; - } - - let keys_elements = - (keys_arr as *const u8).add(std::mem::size_of::()) as *const f64; - let mut prefixes: Vec = Vec::with_capacity(shape_fields as usize); - for f in 0..shape_fields { - let key_bits = (*keys_elements.add(f as usize)).to_bits(); - let key_tag = key_bits & 0xFFFF_0000_0000_0000; - let key_ptr = if key_tag == STRING_TAG || key_tag == POINTER_TAG { - (key_bits & POINTER_MASK) as *const StringHeader - } else { - key_bits as *const StringHeader - }; - let key_str = str_from_header(key_ptr)?; - let needs_escape = key_str.bytes().any(|b| b == b'"' || b == b'\\' || b < 0x20); - let mut prefix = String::with_capacity(key_str.len() + 4); - prefix.push(if f == 0 { '{' } else { ',' }); - if needs_escape { - write_escaped_string(&mut prefix, key_str); - } else { - prefix.push('"'); - prefix.push_str(key_str); - prefix.push('"'); - } - prefix.push(':'); - prefixes.push(prefix); - } - - // Sample first element to decide whether every field slot is already - // a primitive (number/bool/null/string). When true, per-element emit - // can skip the undefined/closure pre-scan. - let fields_ptr = - (first_ptr as *const u8).add(std::mem::size_of::()) as *const f64; - let mut primitive_only = true; - for f in 0..shape_fields { - let fb = (*fields_ptr.add(f as usize)).to_bits(); - if fb == TAG_UNDEFINED || (fb & 0xFFFF_0000_0000_0000) == POINTER_TAG { - primitive_only = false; - break; - } - } - - Some(ShapeTemplate { - keys_arr, - prefixes, - shape_fields, - primitive_only, - }) -} - -/// Record field `f`'s property name (from the shape template's shared -/// `keys_arr`) as the pending `toJSON` key before recursing into that field -/// (#5909). Mirrors the key decode in `build_shape_prefix_template`. -#[inline] -unsafe fn set_to_json_key_for_template_field(template: &ShapeTemplate, f: usize) { - let keys_elements = (template.keys_arr as *const u8) - .add(std::mem::size_of::()) as *const f64; - let key_bits = (*keys_elements.add(f)).to_bits(); - let key_tag = key_bits & 0xFFFF_0000_0000_0000; - let key_ptr = if key_tag == STRING_TAG || key_tag == POINTER_TAG { - (key_bits & POINTER_MASK) as *const StringHeader - } else { - key_bits as *const StringHeader - }; - set_to_json_key_str(str_from_header(key_ptr).unwrap_or("")); -} - -/// Fast emission path for an object element that matches the cached shape -/// template. Returns `true` when the element was emitted via the template; -/// `false` when the element diverges (different shape, skippable field, or -/// has a `toJSON` that must produce the replacement value). On `false` the -/// buffer is unchanged — the caller is responsible for falling back. -pub(crate) unsafe fn try_emit_shape_element( - elem_bits: u64, - template: &ShapeTemplate, - buf: &mut String, - depth: u32, -) -> bool { - let tag = elem_bits & 0xFFFF_0000_0000_0000; - let elem_ptr = if tag == POINTER_TAG { - (elem_bits & POINTER_MASK) as *const u8 - } else if is_raw_pointer(elem_bits) { - elem_bits as *const u8 - } else { - return false; - }; - if gc_obj_type(elem_ptr) != crate::gc::GC_TYPE_OBJECT { - return false; - } - let obj = elem_ptr as *const crate::ObjectHeader; - if (*obj).keys_array != template.keys_arr { - return false; - } - - let fields_ptr = - (elem_ptr as *const u8).add(std::mem::size_of::()) as *const f64; - let shape_fields = template.shape_fields; - let prefixes = template.prefixes.as_slice(); - - // Primitive-only fast path (common case for JSON.parse output): skip - // the undefined/closure pre-scan and trust that the sampled element 0 - // was representative. The emit loop handles stray POINTER_TAG values - // via `stringify_value_depth`; a stray UNDEFINED is rare enough that - // we save `buf.len()` pre-emit and roll back on detection. - if template.primitive_only { - let save_pos = buf.len(); - for f in 0..shape_fields as usize { - let field_val = *fields_ptr.add(f); - let fb = field_val.to_bits(); - // UNDEFINED desyncs comma placement → roll back and let the - // slow object path emit this element correctly. - if fb == TAG_UNDEFINED { - buf.truncate(save_pos); - return false; - } - buf.push_str(&prefixes[f]); - let vtag = fb & 0xFFFF_0000_0000_0000; - if fb == TAG_NULL { - buf.push_str("null"); - } else if fb == TAG_TRUE { - buf.push_str("true"); - } else if fb == TAG_FALSE { - buf.push_str("false"); - } else if vtag == STRING_TAG { - let str_ptr = (fb & POINTER_MASK) as *const StringHeader; - if let Some(s) = str_from_header(str_ptr) { - write_escaped_string(buf, s); - } else { - buf.push_str("null"); - } - } else if vtag == crate::value::SHORT_STRING_TAG { - let jsval = JSValue::from_bits(fb); - let mut scratch = [0u8; crate::value::SHORT_STRING_MAX_LEN]; - let n = jsval.short_string_to_buf(&mut scratch); - if let Ok(s) = std::str::from_utf8(&scratch[..n]) { - write_escaped_string(buf, s); - } else { - buf.push_str("null"); - } - } else if vtag == POINTER_TAG || is_raw_pointer(fb) { - set_to_json_key_for_template_field(template, f); - stringify_value_depth(field_val, TYPE_UNKNOWN, buf, depth + 1); - } else { - // A BigInt field reaches `serialize_bigint` via `write_number`, - // which reads the pending `toJSON` key — record it first (#5909). - if vtag == BIGINT_TAG { - set_to_json_key_for_template_field(template, f); - } - write_number(buf, field_val); - } - } - buf.push('}'); - return true; - } - - // General path: template contains (or may contain) pointer/undefined - // fields. Pre-scan to honor JSON spec (skip undefined, skip closures, - // respect toJSON). - let mut has_pointer_fields = false; - for f in 0..shape_fields as usize { - let fb = (*fields_ptr.add(f)).to_bits(); - if fb == TAG_UNDEFINED { - return false; - } - if (fb & 0xFFFF_0000_0000_0000) == POINTER_TAG { - has_pointer_fields = true; - if is_closure_value(fb) || is_symbol_value(fb) { - return false; - } - } - } - if has_pointer_fields { - if let Some(to_json_val) = object_get_to_json(elem_ptr) { - arm_to_json_result_guard(to_json_val); - stringify_value_depth(to_json_val, TYPE_UNKNOWN, buf, depth + 1); - SUPPRESS_NEXT_TO_JSON.with(|c| c.set(false)); - return true; - } - } - for f in 0..shape_fields as usize { - buf.push_str(&prefixes[f]); - let field_val = *fields_ptr.add(f); - let fb = field_val.to_bits(); - let vtag = fb & 0xFFFF_0000_0000_0000; - if fb == TAG_NULL { - buf.push_str("null"); - } else if fb == TAG_TRUE { - buf.push_str("true"); - } else if fb == TAG_FALSE { - buf.push_str("false"); - } else if vtag == STRING_TAG { - let str_ptr = (fb & POINTER_MASK) as *const StringHeader; - if let Some(s) = str_from_header(str_ptr) { - write_escaped_string(buf, s); - } else { - buf.push_str("null"); - } - } else if vtag == crate::value::SHORT_STRING_TAG { - let jsval = JSValue::from_bits(fb); - let mut scratch = [0u8; crate::value::SHORT_STRING_MAX_LEN]; - let n = jsval.short_string_to_buf(&mut scratch); - if let Ok(s) = std::str::from_utf8(&scratch[..n]) { - write_escaped_string(buf, s); - } else { - buf.push_str("null"); - } - } else if vtag == POINTER_TAG || is_raw_pointer(fb) { - set_to_json_key_for_template_field(template, f); - stringify_value_depth(field_val, TYPE_UNKNOWN, buf, depth + 1); - } else { - // A BigInt field reaches `serialize_bigint` via `write_number`, - // which reads the pending `toJSON` key — record it first (#5909). - if vtag == BIGINT_TAG { - set_to_json_key_for_template_field(template, f); - } - write_number(buf, field_val); - } - } - buf.push('}'); - true -} - /// Depth-aware variant of stringify_array for recursive calls. pub(crate) unsafe fn stringify_array_depth(ptr: *const u8, buf: &mut String, depth: u32) { // Issue #2021: an array that has grown past its initial inline capacity diff --git a/crates/perry-runtime/src/json/stringify_shape_template.rs b/crates/perry-runtime/src/json/stringify_shape_template.rs new file mode 100644 index 0000000000..53e9c22a33 --- /dev/null +++ b/crates/perry-runtime/src/json/stringify_shape_template.rs @@ -0,0 +1,442 @@ +//! Homogeneous-array shape templates for `JSON.stringify`. +//! +//! When every element of an array shares the same `keys_array` pointer (same +//! shape), the key portion of each field can be pre-formatted once and reused +//! across every element. This module owns that template: its cache +//! (`SHAPE_CACHE`), the builder, and the per-element emitter. Split out of +//! `stringify.rs` to keep that file under the 2000-line lint gate. +//! +//! The single invariant everything here turns on: `keys_len` is the LOGICAL +//! property count and `field_count` is PHYSICAL (capped at the object's inline +//! slot allocation). Slots at or above `max(field_count, INLINE_SLOT_FLOOR)` +//! live in overflow storage, so they must be read through +//! `js_object_get_field`, never off the inline `fields_ptr` (#7264). + +use super::stringify::{ + arm_to_json_result_guard, is_closure_value, is_symbol_value, object_get_to_json, + stringify_value_depth, +}; +use super::*; +use crate::{JSValue, StringHeader}; + +/// Cached shape template for a homogeneous array of objects. +pub(crate) struct ShapeTemplate { + pub(crate) keys_arr: *mut crate::ArrayHeader, + pub(crate) prefixes: Vec, + pub(crate) shape_fields: u32, + /// True when element 0's fields are all primitives (no POINTER_TAG / + /// UNDEFINED). Lets the emit path skip its per-element pre-scan. + pub(crate) primitive_only: bool, +} + +/// Look up (or build & insert) the shape template for an object. Returns +/// `None` if the object isn't templatable (no keys array, too many fields, +/// malformed key strings) or if the cache is full and missed. +/// +/// Returns a raw pointer because lifetimes can't survive the TLS borrow. +/// The pointer stays valid until the next `take_shape_cache` (top-level +/// entry/exit) — within one stringify traversal we only `push`, and +/// `Box`'s heap address is stable across `Vec` growth. +#[inline] +pub(crate) unsafe fn shape_template_for(obj_ptr: *const u8) -> Option<*const ShapeTemplate> { + let obj = obj_ptr as *const crate::ObjectHeader; + let keys_arr = (*obj).keys_array; + if keys_arr.is_null() { + return None; + } + + SHAPE_CACHE.with(|c| { + // Fast path: linear scan from the back — recently-used entries + // cluster there for typical traversal orders (shape A's elements + // recurse into shape B repeatedly). + { + let cache = c.borrow(); + for entry in cache.iter().rev() { + if entry.0 == keys_arr { + return Some(&*entry.1 as *const ShapeTemplate); + } + } + if cache.len() >= SHAPE_CACHE_CAP { + return None; + } + } + + // Miss — build, insert, return raw pointer to the boxed template. + let elem_bits = make_pointer_bits(obj_ptr); + let template = build_shape_prefix_template(elem_bits)?; + let mut cache = c.borrow_mut(); + // Re-check cap after the borrow round-trip (a recursive call + // during template build could have filled the cache). + if cache.len() >= SHAPE_CACHE_CAP { + return None; + } + cache.push((keys_arr, Box::new(template))); + Some(&*cache.last().unwrap().1 as *const ShapeTemplate) + }) +} + +/// Build a per-shape key-prefix template for a homogeneous array of objects. +/// +/// When every element of an array shares the same `keys_array` pointer (same +/// shape), we can pre-format the key portion of each field once and reuse it +/// across every element — turning the per-field key lookup (load key f64, +/// extract pointer, `str_from_header`, 3 `push`/`push_str` calls) into a +/// single `push_str` of a cached prefix. +/// +/// Prefix layout for N fields with keys k0..kN-1: +/// `prefixes[0] = "{\"k0\":"` (opening brace fused with first key) +/// `prefixes[f>0] = ",\"kf\":"` (comma fused with key) +/// Close with `}`. This compresses ~7 per-field write ops down to ~2. +/// +/// Returns `None` when the first element isn't a regular object, the keys +/// array is invalid, or any key string is malformed — callers fall back to +/// the generic slow path in that case. +pub(crate) unsafe fn build_shape_prefix_template(first_elem_bits: u64) -> Option { + let tag = first_elem_bits & 0xFFFF_0000_0000_0000; + let first_ptr = if tag == POINTER_TAG { + (first_elem_bits & POINTER_MASK) as *const u8 + } else if is_raw_pointer(first_elem_bits) { + first_elem_bits as *const u8 + } else { + return None; + }; + // Issue #639: Buffer / Uint8Array have no GcHeader, so `gc_obj_type` + // would read 8 bytes before the BufferHeader (unrelated memory) and + // could randomly return GC_TYPE_OBJECT. Bail to the per-element + // slow path which dispatches via `is_registered_buffer`. + if crate::buffer::is_registered_buffer(first_ptr as usize) { + return None; + } + if gc_obj_type(first_ptr) != crate::gc::GC_TYPE_OBJECT { + return None; + } + let obj = first_ptr as *const crate::ObjectHeader; + // A non-zero `class_id` means this object may resolve `toJSON` (or other + // serialization-affecting methods) on its prototype / class vtable — which + // the prefix-template emit path can't see (it only inspects own fields). + // Bail to the per-element slow path (`stringify_object_inner`), which + // probes the prototype chain via `object_get_to_json`. Plain data object + // literals and `JSON.parse` output carry `class_id == 0`, so the + // array-of-objects fast path is unaffected for them. (#321 — a homogeneous + // array of `class { toJSON() {…} }` instances must honour the prototype + // `toJSON`.) + if (*obj).class_id != 0 { + return None; + } + // #6519: a URL instance is a class_id-0 object but must serialize as its + // `href` string (handled by `stringify_object_inner`'s URL branch), never + // as a templated dump of its 12 internal fields (which would also walk the + // `searchParams` back-reference and throw). Bail so an array whose first + // element is a URL routes every element through the per-element slow path. + if crate::url::is_url_object_shape(obj as *mut crate::ObjectHeader) { + return None; + } + let keys_arr = (*obj).keys_array; + if keys_arr.is_null() { + return None; + } + // #2438: array-index keys must enumerate first in ascending numeric order, + // which the insertion-ordered prefix template can't express. Bail to the + // generic slow path (`stringify_object_inner`), which reorders per spec. + if crate::object::keys_contain_array_index(keys_arr) { + return None; + } + // `keys_len` is authoritative — it is the LOGICAL property count, and the + // only field that agrees with `Object.keys` / the slow object path. + // `field_count` is a PHYSICAL count: it never exceeds the object's inline + // slot allocation, so it under-counts any object grown past + // `INLINE_SLOT_FLOOR` by name (`js_object_set_field_by_name` parks slots + // ≥ `max(field_count, INLINE_SLOT_FLOOR)` in overflow storage and leaves + // `field_count` pinned at the floor). `JSON.parse`'s tape materializer is + // exactly that shape: it allocates with `js_object_alloc(0, 0)` and adds + // every property by name, so a 6-key record reports `field_count == 4`. + // + // The old `min(keys_len, field_count)` therefore truncated EVERY element of + // a homogeneous array to the first 4 properties with no diagnostic — silent + // data loss in `JSON.stringify(JSON.parse(x))` (#7264). Latent since the + // template landed (v0.5.65); exposed for ordinary 5–8-field records when + // #6712 lowered `INLINE_SLOT_FLOOR` from 8 to 4. + // + // `min` was never needed for the opposite skew either: a pre-sized object + // (`js_object_alloc(0, 8)` holding 2 real keys) has `field_count > keys_len`, + // and `keys_len` already stops at the last real key. Slots at or above the + // inline limit are read through `template_field_bits`, which routes them to + // `js_object_get_field`'s overflow fallback. + let shape_fields = (*keys_arr).length; + if shape_fields == 0 || shape_fields > 32 { + return None; + } + + let keys_elements = + (keys_arr as *const u8).add(std::mem::size_of::()) as *const f64; + let mut prefixes: Vec = Vec::with_capacity(shape_fields as usize); + for f in 0..shape_fields { + let key_bits = (*keys_elements.add(f as usize)).to_bits(); + let key_tag = key_bits & 0xFFFF_0000_0000_0000; + let key_ptr = if key_tag == STRING_TAG || key_tag == POINTER_TAG { + (key_bits & POINTER_MASK) as *const StringHeader + } else { + key_bits as *const StringHeader + }; + let key_str = str_from_header(key_ptr)?; + let needs_escape = key_str.bytes().any(|b| b == b'"' || b == b'\\' || b < 0x20); + let mut prefix = String::with_capacity(key_str.len() + 4); + prefix.push(if f == 0 { '{' } else { ',' }); + if needs_escape { + write_escaped_string(&mut prefix, key_str); + } else { + prefix.push('"'); + prefix.push_str(key_str); + prefix.push('"'); + } + prefix.push(':'); + prefixes.push(prefix); + } + + // Sample first element to decide whether every field slot is already + // a primitive (number/bool/null/string). When true, per-element emit + // can skip the undefined/closure pre-scan. + let mut primitive_only = true; + for f in 0..shape_fields { + let fb = template_field_bits(obj, f); + if fb == TAG_UNDEFINED || (fb & 0xFFFF_0000_0000_0000) == POINTER_TAG { + primitive_only = false; + break; + } + } + + Some(ShapeTemplate { + keys_arr, + prefixes, + shape_fields, + primitive_only, + }) +} + +/// Physical inline-slot limit of `obj` — the number of field slots the +/// allocator actually reserved. Mirrors `stringify_object_inner`'s +/// `alloc_limit` and every other access path (`object/alloc.rs`); slots at or +/// above it live in overflow storage, not in the inline region. +#[inline] +unsafe fn object_alloc_limit(obj: *const crate::ObjectHeader) -> u32 { + std::cmp::max((*obj).field_count, crate::object::INLINE_SLOT_FLOOR as u32) +} + +/// Read shape-template field slot `f` of `obj`: inline when it fits in the +/// physical allocation, overflow storage otherwise. +/// +/// Reading `fields_ptr[f]` unconditionally is only correct while +/// `shape_fields <= alloc_limit`. It is not: a `JSON.parse` tape object is +/// allocated with `field_count = 0` and grown by name, so a 6-key record has +/// 4 inline slots and 2 overflow slots. Blind inline reads dropped those two +/// on every element of an array (#7264); worse, reading past the allocation +/// would walk into the adjacent arena object. +#[inline] +unsafe fn template_field_bits(obj: *const crate::ObjectHeader, f: u32) -> u64 { + if f < object_alloc_limit(obj) { + let fields_ptr = + (obj as *const u8).add(std::mem::size_of::()) as *const f64; + (*fields_ptr.add(f as usize)).to_bits() + } else { + crate::object::js_object_get_field(obj, f).bits() + } +} + +/// Record field `f`'s property name (from the shape template's shared +/// `keys_arr`) as the pending `toJSON` key before recursing into that field +/// (#5909). Mirrors the key decode in `build_shape_prefix_template`. +#[inline] +unsafe fn set_to_json_key_for_template_field(template: &ShapeTemplate, f: usize) { + let keys_elements = (template.keys_arr as *const u8) + .add(std::mem::size_of::()) as *const f64; + let key_bits = (*keys_elements.add(f)).to_bits(); + let key_tag = key_bits & 0xFFFF_0000_0000_0000; + let key_ptr = if key_tag == STRING_TAG || key_tag == POINTER_TAG { + (key_bits & POINTER_MASK) as *const StringHeader + } else { + key_bits as *const StringHeader + }; + set_to_json_key_str(str_from_header(key_ptr).unwrap_or("")); +} + +/// Fast emission path for an object element that matches the cached shape +/// template. Returns `true` when the element was emitted via the template; +/// `false` when the element diverges (different shape, skippable field, or +/// has a `toJSON` that must produce the replacement value). On `false` the +/// buffer is unchanged — the caller is responsible for falling back. +pub(crate) unsafe fn try_emit_shape_element( + elem_bits: u64, + template: &ShapeTemplate, + buf: &mut String, + depth: u32, +) -> bool { + let tag = elem_bits & 0xFFFF_0000_0000_0000; + let elem_ptr = if tag == POINTER_TAG { + (elem_bits & POINTER_MASK) as *const u8 + } else if is_raw_pointer(elem_bits) { + elem_bits as *const u8 + } else { + return false; + }; + if gc_obj_type(elem_ptr) != crate::gc::GC_TYPE_OBJECT { + return false; + } + let obj = elem_ptr as *const crate::ObjectHeader; + if (*obj).keys_array != template.keys_arr { + return false; + } + + let fields_ptr = + (elem_ptr as *const u8).add(std::mem::size_of::()) as *const f64; + let shape_fields = template.shape_fields; + let prefixes = template.prefixes.as_slice(); + // Per ELEMENT, not per template: two objects can share a `keys_array` + // (same shape) while one was pre-sized and the other grown by name, so the + // inline/overflow split has to be re-derived from this element's own + // header. Hoisted out of the emit loops — the common case is + // `shape_fields <= alloc_limit` and the branch never taken. + let alloc_limit = object_alloc_limit(obj); + let field_bits_at = |f: usize| -> u64 { + if (f as u32) < alloc_limit { + (*fields_ptr.add(f)).to_bits() + } else { + crate::object::js_object_get_field(obj, f as u32).bits() + } + }; + + // Primitive-only fast path (common case for JSON.parse output): skip + // the undefined/closure pre-scan and trust that the sampled element 0 + // was representative. The emit loop handles stray POINTER_TAG values + // via `stringify_value_depth`; a stray UNDEFINED / closure / symbol is + // rare enough that we save `buf.len()` pre-emit and roll back on + // detection. + if template.primitive_only { + let save_pos = buf.len(); + for f in 0..shape_fields as usize { + let fb = field_bits_at(f); + let field_val = f64::from_bits(fb); + // UNDEFINED desyncs comma placement → roll back and let the + // slow object path emit this element correctly. + if fb == TAG_UNDEFINED { + buf.truncate(save_pos); + return false; + } + // Same for a function- or symbol-valued property: SerializeJSONObject + // OMITS the key entirely, and this loop would emit `"k":null` (the + // `stringify_value_depth` rendering of a closure) with the key + // prefix already written. Element 0 having been all-primitives says + // nothing about element N — the general path below pre-scans for + // exactly this and bails; the fast path did not, so a later element + // that took a function value rendered a spurious `null` member. + if (fb & 0xFFFF_0000_0000_0000) == POINTER_TAG + && (is_closure_value(fb) || is_symbol_value(fb)) + { + buf.truncate(save_pos); + return false; + } + buf.push_str(&prefixes[f]); + let vtag = fb & 0xFFFF_0000_0000_0000; + if fb == TAG_NULL { + buf.push_str("null"); + } else if fb == TAG_TRUE { + buf.push_str("true"); + } else if fb == TAG_FALSE { + buf.push_str("false"); + } else if vtag == STRING_TAG { + let str_ptr = (fb & POINTER_MASK) as *const StringHeader; + if let Some(s) = str_from_header(str_ptr) { + write_escaped_string(buf, s); + } else { + buf.push_str("null"); + } + } else if vtag == crate::value::SHORT_STRING_TAG { + let jsval = JSValue::from_bits(fb); + let mut scratch = [0u8; crate::value::SHORT_STRING_MAX_LEN]; + let n = jsval.short_string_to_buf(&mut scratch); + if let Ok(s) = std::str::from_utf8(&scratch[..n]) { + write_escaped_string(buf, s); + } else { + buf.push_str("null"); + } + } else if vtag == POINTER_TAG || is_raw_pointer(fb) { + set_to_json_key_for_template_field(template, f); + stringify_value_depth(field_val, TYPE_UNKNOWN, buf, depth + 1); + } else { + // A BigInt field reaches `serialize_bigint` via `write_number`, + // which reads the pending `toJSON` key — record it first (#5909). + if vtag == BIGINT_TAG { + set_to_json_key_for_template_field(template, f); + } + write_number(buf, field_val); + } + } + buf.push('}'); + return true; + } + + // General path: template contains (or may contain) pointer/undefined + // fields. Pre-scan to honor JSON spec (skip undefined, skip closures, + // respect toJSON). + let mut has_pointer_fields = false; + for f in 0..shape_fields as usize { + let fb = field_bits_at(f); + if fb == TAG_UNDEFINED { + return false; + } + if (fb & 0xFFFF_0000_0000_0000) == POINTER_TAG { + has_pointer_fields = true; + if is_closure_value(fb) || is_symbol_value(fb) { + return false; + } + } + } + if has_pointer_fields { + if let Some(to_json_val) = object_get_to_json(elem_ptr) { + arm_to_json_result_guard(to_json_val); + stringify_value_depth(to_json_val, TYPE_UNKNOWN, buf, depth + 1); + SUPPRESS_NEXT_TO_JSON.with(|c| c.set(false)); + return true; + } + } + for f in 0..shape_fields as usize { + buf.push_str(&prefixes[f]); + let fb = field_bits_at(f); + let field_val = f64::from_bits(fb); + let vtag = fb & 0xFFFF_0000_0000_0000; + if fb == TAG_NULL { + buf.push_str("null"); + } else if fb == TAG_TRUE { + buf.push_str("true"); + } else if fb == TAG_FALSE { + buf.push_str("false"); + } else if vtag == STRING_TAG { + let str_ptr = (fb & POINTER_MASK) as *const StringHeader; + if let Some(s) = str_from_header(str_ptr) { + write_escaped_string(buf, s); + } else { + buf.push_str("null"); + } + } else if vtag == crate::value::SHORT_STRING_TAG { + let jsval = JSValue::from_bits(fb); + let mut scratch = [0u8; crate::value::SHORT_STRING_MAX_LEN]; + let n = jsval.short_string_to_buf(&mut scratch); + if let Ok(s) = std::str::from_utf8(&scratch[..n]) { + write_escaped_string(buf, s); + } else { + buf.push_str("null"); + } + } else if vtag == POINTER_TAG || is_raw_pointer(fb) { + set_to_json_key_for_template_field(template, f); + stringify_value_depth(field_val, TYPE_UNKNOWN, buf, depth + 1); + } else { + // A BigInt field reaches `serialize_bigint` via `write_number`, + // which reads the pending `toJSON` key — record it first (#5909). + if vtag == BIGINT_TAG { + set_to_json_key_for_template_field(template, f); + } + write_number(buf, field_val); + } + } + buf.push('}'); + true +} diff --git a/test-files/test_gap_json_array_element_overflow_fields.ts b/test-files/test_gap_json_array_element_overflow_fields.ts new file mode 100644 index 0000000000..0d8c5f77b4 --- /dev/null +++ b/test-files/test_gap_json_array_element_overflow_fields.ts @@ -0,0 +1,118 @@ +// JSON.stringify of an array whose elements carry more properties than the +// object's INLINE SLOT allocation must emit every property, not just the ones +// that happen to sit in inline slots. +// +// Perry's array fast path builds one shape template from element 0 and reuses +// its pre-formatted key prefixes for every element. It sized that template +// from `min(keys_len, field_count)`. `keys_len` is the LOGICAL property count; +// `field_count` is PHYSICAL — capped at the object's inline slot allocation. +// An object grown by name past INLINE_SLOT_FLOOR (4) keeps `field_count` +// pinned at the floor and parks the rest of its values in overflow storage. +// +// `JSON.parse`'s lazy tape materializer builds exactly that shape, so +// `JSON.stringify(JSON.parse(blob))` silently dropped every property past the +// 4th from EVERY record, as soon as any property of any element was read +// (the read is what forces materialization). Five-field API records — the +// single most common JSON shape there is — lost a field with no diagnostic. +// (#7264; latent since v0.5.65, exposed by #6712 lowering the floor 8 → 4.) +// +// Validated byte-for-byte against `node --experimental-strip-types`. + +// (1) Field-count sweep across the inline floor. The truncation was invisible +// at <= 4 fields and produced IDENTICAL output for 5, 6 and 8 — the tell +// that emission stopped at the floor. +for (const n of [1, 2, 3, 4, 5, 6, 8, 12]) { + const items: any[] = []; + for (let i = 0; i < 32; i++) { + const o: any = {}; + for (let f = 0; f < n; f++) o["f" + f] = i + f; + items.push(o); + } + const parsed = JSON.parse(JSON.stringify(items)); + const touched = parsed[0]["f0"]; // ONE read forces materialization + const out = JSON.stringify(parsed); + console.log(n + " " + touched + " " + out.length + " " + out.slice(0, 72)); +} + +// (2) Reading ONE property of ONE element corrupted all 32 records; so did a +// loop over every element; the untouched array was correct. All three +// must agree. +function sixField(): any[] { + const items: any[] = []; + for (let i = 0; i < 32; i++) { + items.push({ f0: i, f1: i + 1, f2: i + 2, f3: i + 3, f4: i + 4, f5: i + 5 }); + } + return items; +} +const blob = JSON.stringify(sixField()); + +const untouched = JSON.parse(blob); +console.log("untouched " + JSON.stringify(untouched).length); + +const oneRead = JSON.parse(blob); +const single = oneRead[0].f0; +console.log("one-read " + single + " " + JSON.stringify(oneRead).length); + +const loopRead = JSON.parse(blob); +let sum = 0; +for (let i = 0; i < loopRead.length; i++) sum += loopRead[i].f2; +console.log("loop-read " + sum + " " + JSON.stringify(loopRead).length); + +console.log("agree " + (JSON.stringify(untouched) === blob) + + " " + (JSON.stringify(oneRead) === blob) + + " " + (JSON.stringify(loopRead) === blob)); + +// (3) The whole-array result must agree with the per-element results and with +// Object.keys. Only the ARRAY path was wrong, which is what made the bug +// so hard to see: every element-level probe reported the truth. +const probe = JSON.parse(blob); +const z = probe[0].f0; +console.log("keys " + Object.keys(probe[0]).length + " " + z); +console.log("element " + JSON.stringify(probe[0])); +console.log("mapped " + (JSON.stringify(probe) === "[" + probe.map((o: any) => JSON.stringify(o)).join(",") + "]")); + +// (4) The originally-reported shape: the 5th property is a nested object and +// the 4th an array, so the dropped field was a whole subtree. +const records: any[] = []; +for (let i = 0; i < 32; i++) { + records.push({ + id: i, + name: "item_" + i, + value: i * 3, + tags: ["tag_" + (i % 10), "tag_" + (i % 5)], + nested: { x: i, y: i * 2 }, + }); +} +const nestedBlob = JSON.stringify(records); +const nestedParsed = JSON.parse(nestedBlob); +let xs = 0; +for (let i = 0; i < nestedParsed.length; i++) xs += nestedParsed[i].nested.x; +console.log("nested " + xs + " " + (JSON.stringify(nestedParsed) === nestedBlob)); +console.log("nested-tail " + JSON.stringify(nestedParsed).slice(-64)); + +// (5) The template's non-primitive fallbacks must still fire for overflow +// slots: `undefined` is skipped, a function is skipped, and a `toJSON` +// found past the floor replaces the whole element. +const mixed = JSON.parse(blob); +const mixedZ = mixed[0].f0; +mixed[1].f5 = undefined; +mixed[2].f4 = function ignored() {}; +mixed[3].toJSON = function () { return "replaced"; }; +console.log("mixed " + mixedZ + " " + JSON.stringify(mixed.slice(0, 5))); + +// (6) The opposite skew must not regress: an object holding FEWER properties +// than its inline allocation must still stop at its last real key. +const sparse: any[] = []; +for (let i = 0; i < 32; i++) sparse.push({ a: i, b: i + 1 }); +const sparseParsed = JSON.parse(JSON.stringify(sparse)); +const sparseZ = sparseParsed[0].a; +console.log("sparse " + sparseZ + " " + JSON.stringify(sparseParsed).slice(0, 40)); + +// (7) Heterogeneous array: element 0 seeds the template, later elements have a +// different shape and must fall back per element rather than being forced +// through the template. +const hetero = JSON.parse( + '[{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6},{"a":1,"b":2},{"a":1,"b":2,"c":3,"d":4,"e":5,"f":6,"g":7}]', +); +const heteroZ = hetero[0].a; +console.log("hetero " + heteroZ + " " + JSON.stringify(hetero));