From 29217e85ade8aef78615c7329583e90f3e050f86 Mon Sep 17 00:00:00 2001 From: Chris Lundquist Date: Wed, 10 Jun 2026 11:41:05 -0700 Subject: [PATCH] =?UTF-8?q?feat(pz2):=20frozen=20shared=20match-finder=20?= =?UTF-8?q?=E2=80=94=20dict-tier=20encode=20at=203x=20spike=20speed,=20sam?= =?UTF-8?q?e=20ratio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The encode-side prerequisite for the Pz2d dict tier (#146 findings): - lz77::FrozenDict: immutable hash-chain tables built ONCE over a dict prefix (one insert per position, no chain walks), Arc-shared by all workers. Dict-relative coordinates; the parse input must carry the dict as its prefix (worker arena = dict||block) so compares read one buffer. - HashChainFinder::set_frozen_dict + a second chain walk in find_best over the frozen tables after the block-local walk (shared chain budget, recency first). The pos >= dict.len guard makes every read in-bounds regardless of caller behavior. - lzseq::tokenize_with_dict(input, start, dict, config): parse starts at the dict boundary — no dict re-parse, no token-skip/straddle handling (tokenize_with_config delegates with (0, None)). - pz2::encode_with_frozen_dict (wire-writing extracted into a shared encode_sequences; output decodes with the existing decode_with_prefix). Measured (pz2_dict_probe --frozen, per-32MiB-segment, blob): - ratio IDENTICAL to the re-parse spike: 30.475% at 16 MiB dict (-0.57pp vs no dict, ~0.9pp under pzstd-3) - encode 70.5s vs the spike's 208.8s ST (3x); remaining cost is the dict chain walks themselves (5.3x no-dict baseline) - a 32-byte weak-local-match gate was measured and REJECTED (-8% time, +0.018pp: most text positions have weak local matches — the walk is inherent; tune via dict chain caps at integration time) Remaining for shipping Pz2d (documented in section 11): container segment framing, 2-wave parallel decode with Arc dict + worker arenas, encode-cost tuning pass. 594 + 741 tests, fmt, clippy clean. Co-Authored-By: Claude Fable 5 --- docs/design-docs/clean-slate-codec.md | 34 ++++--- examples/pz2_dict_probe.rs | 42 +++++++- src/lz77/mod.rs | 136 ++++++++++++++++++++++++++ src/lzseq/mod.rs | 22 ++++- src/pz2.rs | 67 ++++++++++++- 5 files changed, 281 insertions(+), 20 deletions(-) diff --git a/docs/design-docs/clean-slate-codec.md b/docs/design-docs/clean-slate-codec.md index 4e8261e..4594597 100644 --- a/docs/design-docs/clean-slate-codec.md +++ b/docs/design-docs/clean-slate-codec.md @@ -409,14 +409,26 @@ segment tier earning its place in the format. copies split). Naive per-block priming is ruled out by arithmetic: 94 blocks × 16 MiB ≈ 1.5 GB of memcpy ≈ +15-30 ms — would double the 16.8 ms decode wall. -- **Encode (the real blocker):** the spike re-tokenizes dict+block per - block (9× work at 16 MiB — fine for measurement, unshippable). Production - needs a **frozen shared match-finder**: build hash chains over the dict - once (immutable, `Arc`-shared head/prev arrays in dict coordinates), - each worker holds a `dict‖block` arena (dict copied once per worker, so - frozen-table coordinates match and compares never need two-region - logic) and parses starting at `dict_len` — which also eliminates the - spike's token-skipping/straddle handling. `find_best` grows one extra - chain walk over the frozen tables after the block-local walk. -- **Measured payoff (per-segment, 16 MiB dict):** blob 31.04% → **30.48%**, - ~0.9pp under pzstd-3 (31.4%), at unchanged decode parallelism. +- **Encode: the frozen shared match-finder is BUILT and measured** + (`lz77::FrozenDict` + `lzseq::tokenize_with_dict` + + `pz2::encode_with_frozen_dict`): dict chains built once (immutable, + `Arc`-shared, dict-relative coordinates), each worker holds a + `dict‖block` arena (so frozen coordinates match and compares read one + buffer) and parses starting at `dict_len` — no dict re-parse, no + token-skipping/straddle handling. `find_best` grows one extra chain + walk over the frozen tables after the block-local walk, sharing the + chain budget. Probe `--frozen` (per-segment, blob): **identical ratio + to the re-parse spike (30.475%) at 3× its encode speed** (70.5 s vs + 208.8 s ST). Remaining encode cost is the dict chain walks themselves + (5.3× the no-dict baseline at 16 MiB) — a 32-byte weak-local-match + gate was measured and rejected (−8% time, +0.018pp: most text + positions have weak local matches, so the walk is inherent). Tuning + levers for integration: dict-specific chain caps, sampled dict + insertion, 4-8 MiB dicts (4 MiB: −0.31pp at 33.7 s). +- **Measured payoff (per-segment, 16 MiB dict, frozen finder):** blob + 31.04% → **30.48%**, ~0.9pp under pzstd-3 (31.4%), at unchanged decode + parallelism. +- **What remains for shipping `Pz2d`:** container integration only — + segment framing (`[dict_len] +` blocks), 2-wave parallel decode with + the `Arc`-shared dict + worker arenas, encode-side worker arenas, and + the encode-cost tuning pass above. diff --git a/examples/pz2_dict_probe.rs b/examples/pz2_dict_probe.rs index d5cb349..103e969 100644 --- a/examples/pz2_dict_probe.rs +++ b/examples/pz2_dict_probe.rs @@ -41,6 +41,16 @@ fn main() { } else { false }; + // --frozen: per-segment head dict via the production frozen finder + // (FrozenDict built once per segment, parse starts at the dict + // boundary) instead of the spike's full dict+block re-parse. Same wire, + // same decoder; encode cost is the point. + let frozen_mode = if let Some(i) = args.iter().position(|a| a == "--frozen") { + args.remove(i); + true + } else { + false + }; const SEG: usize = 32 << 20; if args.is_empty() { eprintln!("usage: pz2_dict_probe [--head] "); @@ -53,7 +63,9 @@ fn main() { "ratio %", "delta", "enc s", - if seg_mode { + if frozen_mode { + "per-segment FROZEN finder" + } else if seg_mode { "per-segment head dict" } else if head_mode { "fixed head dict" @@ -76,9 +88,35 @@ fn main() { let t = Instant::now(); let mut size = 0usize; let mut start = 0usize; + // Frozen mode: build each segment's dict tables ONCE (the + // production shape — Arc-shared across workers). + let mut frozen: Option<(usize, std::sync::Arc, usize)> = None; while start < data.len() { let end = (start + BLOCK).min(data.len()); - let (enc, prefix): (Vec, &[u8]) = if head_mode || seg_mode { + let (enc, prefix): (Vec, &[u8]) = if frozen_mode { + let seg_base = start - (start % SEG); + let dlen = d.min(start - seg_base); + let dict = &data[seg_base..seg_base + dlen]; + let reuse = matches!(frozen, Some((b, _, l)) if b == seg_base && l == dlen); + if !reuse { + frozen = Some(( + seg_base, + std::sync::Arc::new(pz::lz77::FrozenDict::build( + dict, + config.hash_prefix_len, + )), + dlen, + )); + } + let tables = &frozen.as_ref().unwrap().1; + let mut arena = Vec::with_capacity(dlen + (end - start)); + arena.extend_from_slice(dict); + arena.extend_from_slice(&data[start..end]); + ( + pz::pz2::encode_with_frozen_dict(&arena, tables, &config).expect("encode"), + dict, + ) + } else if head_mode || seg_mode { // Dict = first min(d, start - base) bytes of the file // (--head) or of the block's 32 MiB segment (--seg); // blocks inside the dict region parse cold (they ARE diff --git a/src/lz77/mod.rs b/src/lz77/mod.rs index eebd39a..41d3259 100644 --- a/src/lz77/mod.rs +++ b/src/lz77/mod.rs @@ -233,11 +233,70 @@ pub(crate) struct WideMatch { /// /// Maintains a hash table mapping 3-byte or 4-byte prefixes to positions, /// with chains for collision resolution. Average O(n) complexity. +/// Immutable hash-chain tables over a dictionary prefix, built once and +/// shared read-only (`Arc`) by many parallel block parses (the pz2 dict +/// tier's frozen finder). +/// +/// Positions are dict-relative (`0..len`). The parse input handed to +/// [`HashChainFinder::find_match_wide`] MUST carry the same dict bytes at +/// those positions (i.e. `input = dict ‖ block`), so frozen coordinates need +/// no translation and match compares read one contiguous buffer. The walk +/// in `find_best` additionally guards `pos >= dict.len`, so a misuse cannot +/// read out of bounds — it just finds nothing useful. +pub struct FrozenDict { + /// head[hash] = most recent dict position with this hash (0 = empty or + /// position 0 — the same benign ambiguity the live finder has). + head: Vec, + /// prev[pos] = previous dict position in the chain, indexed directly. + prev: Vec, + len: usize, + hash_prefix_len: u8, +} + +impl FrozenDict { + /// Build chains over `dict` once. Cost is one insert per position + /// (no chain walks), ~hundreds of MB/s; share the result via `Arc`. + pub fn build(dict: &[u8], hash_prefix_len: u8) -> Self { + let mut head = vec![0u32; HASH_SIZE]; + let mut prev = vec![0u32; dict.len()]; + // Same lookahead guard as `insert`: the last 2 positions can't hash. + let hashable = dict.len().saturating_sub(2); + for (pos, slot) in prev.iter_mut().enumerate().take(hashable) { + let h = if hash_prefix_len == 4 { + hash4(dict, pos) + } else { + hash3(dict, pos) + }; + *slot = head[h]; + head[h] = pos as u32; + } + FrozenDict { + head, + prev, + len: dict.len(), + hash_prefix_len, + } + } + + /// Dictionary length in bytes. + pub fn len(&self) -> usize { + self.len + } + + /// Whether the dictionary is empty. + pub fn is_empty(&self) -> bool { + self.len == 0 + } +} + pub(crate) struct HashChainFinder { /// head[hash] = most recent position with this hash, or 0 head: Vec, /// prev[pos % max_window] = previous position in the chain prev: Vec, + /// Optional frozen dictionary chains consulted after the live chain + /// (pz2 dict tier). See [`FrozenDict`] for the coordinate contract. + dict: Option>, /// Cached SIMD dispatcher — resolved once, avoids per-call feature detection. dispatcher: crate::simd::Dispatcher, /// Maximum match length to find. Deflate pipelines use 258 (RFC 1951); @@ -272,6 +331,7 @@ impl HashChainFinder { Self { head: vec![0; HASH_SIZE], prev: vec![0; MAX_WINDOW], + dict: None, dispatcher: crate::simd::Dispatcher::new(), max_match_len: max_match_len as usize, max_chain: max_chain.clamp(1, MAX_CHAIN * 4), @@ -297,6 +357,7 @@ impl HashChainFinder { Self { head: vec![0; HASH_SIZE], prev: vec![0; max_window], + dict: None, dispatcher: crate::simd::Dispatcher::new(), max_match_len: max_match_len as usize, max_chain: MAX_CHAIN, @@ -319,6 +380,7 @@ impl HashChainFinder { Self { head: vec![0; HASH_SIZE], prev: vec![0; max_window], + dict: None, dispatcher: crate::simd::Dispatcher::new(), max_match_len: max_match_len as usize, max_chain: max_chain.clamp(1, MAX_CHAIN * 4), @@ -340,6 +402,7 @@ impl HashChainFinder { Self { head: vec![0; HASH_SIZE], prev: vec![0; max_window], + dict: None, dispatcher: crate::simd::Dispatcher::new(), max_match_len: max_match_len as usize, max_chain: max_chain.clamp(1, MAX_CHAIN * 4), @@ -359,6 +422,17 @@ impl HashChainFinder { } } + /// Install frozen dictionary chains, consulted by `find_best` after the + /// live chain. The parse input must carry the dict bytes as its prefix + /// (see [`FrozenDict`]). + pub(crate) fn set_frozen_dict(&mut self, dict: std::sync::Arc) { + debug_assert_eq!( + dict.hash_prefix_len, self.hash_prefix_len, + "frozen dict and finder must hash identically" + ); + self.dict = Some(dict); + } + /// Dynamically adjust chain depth. Used by adaptive encoding loops. /// Clamps to 1..=MAX_CHAIN*4. pub(crate) fn set_max_chain(&mut self, max_chain: usize) { @@ -439,6 +513,68 @@ impl HashChainFinder { chain_count += 1; } + // Frozen dictionary chains (pz2 dict tier): consulted after the live + // chain — recency first, dictionary as fallback — sharing the same + // chain budget. (A weak-local-match gate at 32 bytes was measured: + // it saves only ~8% encode for +0.018pp on the blob, because most + // positions on text HAVE weak local matches — the walk cost is + // inherent. Tune via dict-specific chain caps at integration time + // if needed.) The `pos >= dict.len` guard makes every read below + // in-bounds regardless of caller behavior: chain_pos < dict.len ≤ + // pos, so chain_pos + probe/cmp_limit < pos + remaining = + // input.len(). + if let Some(dict) = self.dict.as_deref() { + if pos >= dict.len && (best_length as usize) < cmp_limit && chain_count < self.max_chain + { + let mut chain_pos = dict.head[h] as usize; + while chain_pos < dict.len && chain_pos >= min_pos && chain_count < self.max_chain { + if best_length >= MIN_MATCH as u32 { + let probe = best_length as usize; + // SAFETY: chain_pos < dict.len <= pos and probe < + // remaining (see block comment above). + let candidate_probe = unsafe { *input_ptr.add(chain_pos + probe) }; + if candidate_probe != best_probe_byte { + let prev_pos = dict.prev[chain_pos] as usize; + if prev_pos >= chain_pos || prev_pos < min_pos { + break; + } + chain_pos = prev_pos; + chain_count += 1; + continue; + } + } + + // SAFETY: chain_pos + cmp_limit < pos + remaining (block + // comment above); candidates may extend past the dict end + // into the block — valid under the primed-buffer contract. + let match_len = unsafe { + self.dispatcher.compare_bytes_ptr( + input_ptr.add(chain_pos), + pos_ptr, + cmp_limit, + ) + } as u32; + + if match_len > best_length && match_len >= MIN_MATCH as u32 { + best_length = match_len; + best_offset = (pos - chain_pos) as u32; + if best_length as usize >= cmp_limit { + break; + } + // SAFETY: best_length < cmp_limit <= remaining. + best_probe_byte = unsafe { *input_ptr.add(pos + best_length as usize) }; + } + + let prev_pos = dict.prev[chain_pos] as usize; + if prev_pos >= chain_pos || prev_pos < min_pos { + break; + } + chain_pos = prev_pos; + chain_count += 1; + } + } + } + (best_offset, best_length) } diff --git a/src/lzseq/mod.rs b/src/lzseq/mod.rs index c4920cc..8d6e828 100644 --- a/src/lzseq/mod.rs +++ b/src/lzseq/mod.rs @@ -983,11 +983,25 @@ pub fn encode_with_config(input: &[u8], config: &SeqConfig) -> PzResult PzResult> { + tokenize_with_dict(input, 0, None, config) +} + +/// [`tokenize_with_config`] generalized for the pz2 dict tier: tokens are +/// emitted for `input[start..]` only, and an optional frozen dictionary +/// (whose bytes must be `input[..dict.len()]`, with `start >= dict.len()`) +/// extends the match finder's reach without re-inserting the dict — the +/// frozen chains were built once and are shared read-only across workers. +pub(crate) fn tokenize_with_dict( + input: &[u8], + start: usize, + dict: Option>, + config: &SeqConfig, ) -> PzResult> { use crate::lz_token::LzToken; let mut tokens: Vec = Vec::new(); - if input.is_empty() { + if input.len() <= start { return Ok(tokens); } @@ -997,8 +1011,12 @@ pub(crate) fn tokenize_with_config( } else { HashChainFinder::with_window_and_chain(config.max_window, match_limit, config.max_chain) }; + if let Some(d) = dict { + debug_assert!(start >= d.len(), "parse must start at/after the dict end"); + finder.set_frozen_dict(d); + } let mut repeats = RepeatOffsets::new(); - let mut pos: usize = 0; + let mut pos: usize = start; let max_match_len = match_limit as usize; // Adaptive chain depth tracking — identical constants to encode_with_config. diff --git a/src/pz2.rs b/src/pz2.rs index 7a4ee96..e723735 100644 --- a/src/pz2.rs +++ b/src/pz2.rs @@ -722,22 +722,45 @@ pub fn encode_with_prefix(data: &[u8], prefix_len: usize, config: &SeqConfig) -> } build_sequences(&kept) }; + encode_sequences(&seqs, &lits, input.len()) +} + +/// Encode one block from a worker arena (`dict ‖ block`) using frozen +/// dictionary chains built once via [`crate::lz77::FrozenDict::build`] and +/// shared across workers. Unlike [`encode_with_prefix`], the dict is NOT +/// re-parsed — the parse starts at the dict boundary and consults the +/// frozen chains, so per-block encode cost is block-sized. The stream +/// decodes with [`decode_with_prefix`] given the same dict bytes. +pub fn encode_with_frozen_dict( + arena: &[u8], + dict: &std::sync::Arc, + config: &SeqConfig, +) -> PzResult> { + let dict_len = dict.len(); + assert!(dict_len <= arena.len()); + let tokens = + lzseq::tokenize_with_dict(arena, dict_len, Some(std::sync::Arc::clone(dict)), config)?; + let (seqs, lits) = build_sequences(&tokens); + encode_sequences(&seqs, &lits, arena.len() - dict_len) +} - let mut out = Vec::with_capacity(input.len() / 2 + 64); +/// Shared wire writer: sequences + literals → the pz2 block format. +fn encode_sequences(seqs: &[Seq], lits: &[u8], block_len: usize) -> PzResult> { + let mut out = Vec::with_capacity(block_len / 2 + 64); put_u32(&mut out, seqs.len() as u32); put_u32(&mut out, lits.len() as u32); // --- Literal section --- let mut wrote_huff = false; let mut counts = [0u32; 256]; - for &b in &lits { + for &b in lits { counts[b as usize] += 1; } let distinct = counts.iter().filter(|&&c| c > 0).count(); if distinct >= 2 { let lengths = huffman_lengths(&counts); let codes = canonical_codes(&lengths)?; - let lanes = encode_lanes(&lits, &codes); + let lanes = encode_lanes(lits, &codes); let huff_size: usize = 1 + 128 + 4 * NUM_LANES + lanes.iter().map(Vec::len).sum::(); if huff_size < 1 + lits.len() { out.push(LIT_HUFF); @@ -753,7 +776,7 @@ pub fn encode_with_prefix(data: &[u8], prefix_len: usize, config: &SeqConfig) -> } if !wrote_huff { out.push(LIT_RAW); - out.extend_from_slice(&lits); + out.extend_from_slice(lits); } // --- Sequence section --- @@ -764,7 +787,7 @@ pub fn encode_with_prefix(data: &[u8], prefix_len: usize, config: &SeqConfig) -> let mut ml_codes = Vec::with_capacity(n); let mut extras = BitWriter::new(); let mut reps = RepeatOffsets::new(); - for s in &seqs { + for s in seqs { let (c, eb, ev) = vcode(s.lit_run); ll_codes.push(c); extras.write(ev, eb); @@ -1176,6 +1199,40 @@ mod tests { assert_eq!(decode_with_prefix(&enc0, &[], block.len()).unwrap(), block); } + #[test] + fn test_frozen_dict_round_trip() { + use crate::lz77::FrozenDict; + use std::sync::Arc; + + let dict = b"the quick brown fox jumps over the lazy dog. ".repeat(64); + let block = b"the quick brown fox jumps over the lazy dog! ".repeat(80); + let mut arena = dict.clone(); + arena.extend_from_slice(&block); + + let config = SeqConfig::default(); + let frozen = Arc::new(FrozenDict::build(&dict, config.hash_prefix_len)); + let enc = encode_with_frozen_dict(&arena, &frozen, &config).unwrap(); + + // Wire-compatible with the prefix decoder. + let dec = decode_with_prefix(&enc, &dict, block.len()).unwrap(); + assert_eq!(dec, block); + + // The frozen chains must find the dict matches: at least as small + // as the cold encode, in the same family as the re-parse spike. + let cold = encode_with_config(&block, &config).unwrap(); + assert!( + enc.len() < cold.len(), + "frozen-dict encode ({}) not smaller than cold ({})", + enc.len(), + cold.len() + ); + + // Empty dict behaves like a plain encode. + let empty = Arc::new(FrozenDict::build(&[], config.hash_prefix_len)); + let enc0 = encode_with_frozen_dict(&block, &empty, &config).unwrap(); + assert_eq!(decode(&enc0, block.len()).unwrap(), block); + } + #[test] fn test_decode_rejects_garbage() { // Truncations of a valid block must error, never panic.