From 37290f51f83a4afbea610649195a7f7da52398cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 09:30:48 +0100 Subject: [PATCH 01/41] Optimize decompression --- src/decompress.rs | 223 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 164 insertions(+), 59 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 2f4add2..c1fa0d7 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -9,12 +9,67 @@ use crate::MAX_INPUT_SIZE; /// tag byte. const TAG_LOOKUP_TABLE: TagLookupTable = TagLookupTable(tag::TAG_LOOKUP_TABLE); -/// `WORD_MASK` is a map from the size of an integer in bytes to its -/// corresponding on a 32 bit integer. This is used when we need to read an -/// integer and we know there are at least 4 bytes to read from a buffer. In -/// this case, we can read a 32 bit little endian integer and mask out only the -/// bits we need. This in particular saves a branch. -const WORD_MASK: [usize; 5] = [0, 0xFF, 0xFFFF, 0xFFFFFF, 0xFFFFFFFF]; +/// Combined length-and-offset table inspired by C++ snappy's kLengthMinusOffset. +/// +/// For each tag byte, encodes (as i16): +/// - Short literals (len 1-16): `length - 256` (always negative) +/// - Copy-1: `length - (offset_high_bits << 8)` +/// - Copy-2 (len 1-16): `length` +/// - Extended literals, long copies, copy-4: flag value (0x0180) +/// +/// The key property: for common tags (short literals and copies where +/// offset > length), `entry <= ExtractOffset(trailing_bytes, tag_type)`, +/// allowing a single unified code path without branching on tag type. +const LENGTH_MINUS_OFFSET: [i16; 256] = make_length_minus_offset(); + +const fn make_length_minus_offset() -> [i16; 256] { + let mut table = [0i16; 256]; + let mut i = 0usize; + while i < 256 { + let tag = i as u8; + let tag_type = tag & 3; + if tag_type == 0 { + // Literal + let lit_len = (tag >> 2) + 1; + if lit_len <= 16 { + // Common short literal: entry = len - 256 (negative) + table[i] = (lit_len as i16) - 256; + } else { + // Long/extended literal: exit to slow path + table[i] = 0x0180u16 as i16; + } + } else if tag_type == 1 { + // Copy-1: len 4-11, offset has high bits in tag + let len = (4 + ((tag >> 2) & 7)) as i16; + let offset_high = ((tag >> 5) & 7) as i16; + table[i] = len - (offset_high << 8); + } else if tag_type == 2 { + // Copy-2 + let len = (1 + (tag >> 2)) as i16; + if len <= 16 { + table[i] = len; + } else { + // Long copy-2: exit to slow path + table[i] = 0x0180u16 as i16; + } + } else { + // Copy-4: exit to slow path + table[i] = 0x0180u16 as i16; + } + i += 1; + } + table +} + +/// Extract copy offset from trailing bytes using a packed 64-bit mask constant. +/// Returns 0 for literals, low byte for copy-1, full u16 for copy-2, 0 for copy-4. +#[inline(always)] +fn extract_offset(val: u32, tag_type: usize) -> u32 { + const MASKS: u64 = 0x0000FFFF00FF0000u64; + let mask = ((MASKS >> (tag_type * 16)) & 0xFFFF) as u32; + val & mask +} + /// Returns the decompressed size (in bytes) of the compressed bytes given. /// @@ -128,7 +183,99 @@ impl<'s, 'd> Decompress<'s, 'd> { /// This assumes that the header has already been read and that `dst` is /// big enough to store all decompressed bytes. fn decompress(&mut self) -> Result<()> { - while self.s < self.src.len() { + let src_len = self.src.len(); + let dst_len = self.dst.len(); + + // Fast unified loop: processes short literals and copies through + // a single code path using the LENGTH_MINUS_OFFSET table. + // The key insight (from C++ snappy): by encoding literal entries + // as `length - 256`, both literals and copies satisfy + // `entry <= extracted` in the common case, eliminating the need + // to branch on tag type for most of the processing. + 'outer: loop { + // Headroom: need 2 bytes for u16 trailing load, plus up to + // 16 bytes for literal source data. + while self.s + 18 <= src_len && self.d + 16 <= dst_len { + // SAFETY: headroom check guarantees src[s], src[s+1..s+2] + // (for u16 load), and dst[d..d+16] are in bounds. + unsafe { + let tag = *self.src.get_unchecked(self.s); + let tag_type = (tag & 3) as usize; + let entry = *LENGTH_MINUS_OFFSET.get_unchecked(tag as usize); + + let old_s = self.s + 1; + let next = u16::from_le( + (self.src.as_ptr().add(old_s) as *const u16) + .read_unaligned(), + ) as u32; + + let len = (entry & 0xFF) as usize; + if len > 16 { + break; + } + let extracted = extract_offset(next, tag_type) as i16; + + // Catches: pattern-extension copies (offset < len), + // long literals/copies, extended literals, copy-4. + if entry > extracted { + break; + } + + // For copies: len_min_offset = len - full_offset + // For literals: len_min_offset = len - 256 + let len_min_offset = (entry - extracted) as isize; + let full_offset = + (len as isize - len_min_offset) as usize; + + // Validate copy offset: need offset <= d AND offset >= 8 + // (offset < 8 requires pattern-extension handling). + // For literals tag_type == 0, so this is skipped. + if tag_type != 0 + && (full_offset < 8 || full_offset > self.d) + { + break; + } + + // delta = d - full_offset (copy source position) + // For literals: d - 256 (garbage, but masked away) + let delta = self.d.wrapping_sub(full_offset); + + // Branchless source pointer selection via bitmask + let tag_mask = + 0usize.wrapping_sub((tag_type != 0) as usize); + let copy_src = + (self.dst.as_ptr() as usize).wrapping_add(delta); + let lit_src = + self.src.as_ptr() as usize + old_s; + let from = ((copy_src & tag_mask) + | (lit_src & !tag_mask)) + as *const u8; + + let dstp = self.dst.as_mut_ptr().add(self.d); + ptr::copy_nonoverlapping(from, dstp, 8); + ptr::copy_nonoverlapping( + from.add(8), + dstp.add(8), + 8, + ); + + // Advance input: literal skips len data bytes, + // copy skips tag_type trailing bytes. + // This branch is the only tag-type-dependent operation + // AFTER the memory copy, minimizing misprediction cost. + if tag_type == 0 { + self.s = old_s + len; + } else { + self.s = old_s + tag_type; + } + self.d += len; + } + } + + // Slow path: handle one tag, then re-enter the fast loop. + if self.s >= src_len { + break 'outer; + } let byte = self.src[self.s]; self.s += 1; if byte & 0b000000_11 == 0 { @@ -138,9 +285,9 @@ impl<'s, 'd> Decompress<'s, 'd> { self.read_copy(byte)?; } } - if self.d != self.dst.len() { + if self.d != dst_len { return Err(Error::HeaderMismatch { - expected_len: self.dst.len() as u64, + expected_len: dst_len as u64, got_len: self.d as u64, }); } @@ -199,8 +346,9 @@ impl<'s, 'd> Decompress<'s, 'd> { // Since we know there are 4 bytes left to read, read a 32 bit LE // integer and mask away the bits we don't need. let byte_count = len as usize - 60; + let mask = u32::MAX >> ((4 - byte_count as u32) << 3); len = bytes::read_u32_le(&self.src[self.s..]) as u64; - len = (len & (WORD_MASK[byte_count] as u64)) + 1; + len = (len & mask as u64) + 1; self.s += byte_count; } // If there's not enough buffer left to load or store this literal, @@ -255,52 +403,14 @@ impl<'s, 'd> Decompress<'s, 'd> { // loads/stores. if offset >= 8 && len <= 16 && self.d + 16 <= self.dst.len() { unsafe { - // SAFETY: We know dstp points to at least 16 bytes of memory - // from the condition above, and we also know that dstp is - // preceded by at least `offset` bytes from the `d <= offset` - // check above. - // - // We also know that dstp and dstp-8 do not overlap from the - // check above, justifying the use of copy_nonoverlapping. let dstp = self.dst.as_mut_ptr().add(self.d); let srcp = dstp.sub(offset); - // We can't do a single 16 byte load/store because src/dst may - // overlap with each other. Namely, the second copy here may - // copy bytes written in the first copy! ptr::copy_nonoverlapping(srcp, dstp, 8); ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); } - // If we have some wiggle room, try to decompress the copy 16 bytes - // at a time with 128 bit unaligned loads/stores. Remember, we can't - // just do a memcpy because decompressing copies may require copying - // overlapping memory. - // - // We need the extra wiggle room to make effective use of 128 bit - // loads/stores. Even if the store ends up copying more data than we - // need, we're careful to advance `d` by the correct amount at the end. } else if end + 24 <= self.dst.len() { unsafe { - // SAFETY: We know that dstp is preceded by at least `offset` - // bytes from the `d <= offset` check above. - // - // We don't know whether dstp overlaps with srcp, so we start - // by copying from srcp to dstp until they no longer overlap. - // The worst case is when dstp-src = 3 and copy length = 1. The - // first loop will issue these copy operations before stopping: - // - // [-1, 14] -> [0, 15] - // [-1, 14] -> [3, 18] - // [-1, 14] -> [9, 24] - // - // But the copy had length 1, so it was only supposed to write - // to [0, 0]. But the last copy wrote to [9, 24], which is 24 - // extra bytes in dst *beyond* the end of the copy, which is - // guaranteed by the conditional above. - - // Save destination length here to avoid a reborrow UB violation - // under the Tree Borrows model. let dest_len = self.dst.len(); - let mut dstp = self.dst.as_mut_ptr().add(self.d); let mut srcp = dstp.sub(offset); loop { @@ -309,7 +419,6 @@ impl<'s, 'd> Decompress<'s, 'd> { if diff >= 16 { break; } - // srcp and dstp can overlap, so use ptr::copy. debug_assert!(self.d + 16 <= dest_len); ptr::copy(srcp, dstp, 16); self.d += diff as usize; @@ -321,8 +430,6 @@ impl<'s, 'd> Decompress<'s, 'd> { dstp = dstp.add(16); self.d += 16; } - // At this point, `d` is likely wrong. We correct it before - // returning. It's correct value is `end`. } } else { if end > self.dst.len() { @@ -331,8 +438,6 @@ impl<'s, 'd> Decompress<'s, 'd> { dst_len: (self.dst.len() - self.d) as u64, }); } - // Finally, the slow byte-by-byte case, which should only be used - // for the last few bytes of decompression. while self.d != end { self.dst[self.d] = self.dst[self.d - offset]; self.d += 1; @@ -441,12 +546,12 @@ impl TagEntry { // SAFETY: The conditional above guarantees that // src[s..s+4] is valid to read from. let p = src.as_ptr().add(s); - // We use WORD_MASK here to mask out the bits we don't - // need. While we're guaranteed to read 4 valid bytes, - // not all of those bytes are necessarily part of the - // offset. This is the key optimization: we don't need to - // branch on num_tag_bytes. - bytes::loadu_u32_le(p) as usize & WORD_MASK[num_tag_bytes] + // Mask to extract only the bytes we need from the u32. + // num_tag_bytes is 1, 2, or 4 for copy tags, so the + // shift is always 24, 16, or 0 (all < 32). + // This avoids a table lookup on the critical path. + let mask = u32::MAX >> ((4 - num_tag_bytes as u32) << 3); + bytes::loadu_u32_le(p) as usize & mask as usize } } else if num_tag_bytes == 1 { if s >= src.len() { From 6d011ed05fb56641af15e870a8357630705f2c5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 09:42:40 +0100 Subject: [PATCH 02/41] Optimize decompression --- src/decompress.rs | 98 ++--------------------------------------------- 1 file changed, 3 insertions(+), 95 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index c1fa0d7..0714dc7 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -183,99 +183,7 @@ impl<'s, 'd> Decompress<'s, 'd> { /// This assumes that the header has already been read and that `dst` is /// big enough to store all decompressed bytes. fn decompress(&mut self) -> Result<()> { - let src_len = self.src.len(); - let dst_len = self.dst.len(); - - // Fast unified loop: processes short literals and copies through - // a single code path using the LENGTH_MINUS_OFFSET table. - // The key insight (from C++ snappy): by encoding literal entries - // as `length - 256`, both literals and copies satisfy - // `entry <= extracted` in the common case, eliminating the need - // to branch on tag type for most of the processing. - 'outer: loop { - // Headroom: need 2 bytes for u16 trailing load, plus up to - // 16 bytes for literal source data. - while self.s + 18 <= src_len && self.d + 16 <= dst_len { - // SAFETY: headroom check guarantees src[s], src[s+1..s+2] - // (for u16 load), and dst[d..d+16] are in bounds. - unsafe { - let tag = *self.src.get_unchecked(self.s); - let tag_type = (tag & 3) as usize; - let entry = *LENGTH_MINUS_OFFSET.get_unchecked(tag as usize); - - let old_s = self.s + 1; - let next = u16::from_le( - (self.src.as_ptr().add(old_s) as *const u16) - .read_unaligned(), - ) as u32; - - let len = (entry & 0xFF) as usize; - if len > 16 { - break; - } - let extracted = extract_offset(next, tag_type) as i16; - - // Catches: pattern-extension copies (offset < len), - // long literals/copies, extended literals, copy-4. - if entry > extracted { - break; - } - - // For copies: len_min_offset = len - full_offset - // For literals: len_min_offset = len - 256 - let len_min_offset = (entry - extracted) as isize; - let full_offset = - (len as isize - len_min_offset) as usize; - - // Validate copy offset: need offset <= d AND offset >= 8 - // (offset < 8 requires pattern-extension handling). - // For literals tag_type == 0, so this is skipped. - if tag_type != 0 - && (full_offset < 8 || full_offset > self.d) - { - break; - } - - // delta = d - full_offset (copy source position) - // For literals: d - 256 (garbage, but masked away) - let delta = self.d.wrapping_sub(full_offset); - - // Branchless source pointer selection via bitmask - let tag_mask = - 0usize.wrapping_sub((tag_type != 0) as usize); - let copy_src = - (self.dst.as_ptr() as usize).wrapping_add(delta); - let lit_src = - self.src.as_ptr() as usize + old_s; - let from = ((copy_src & tag_mask) - | (lit_src & !tag_mask)) - as *const u8; - - let dstp = self.dst.as_mut_ptr().add(self.d); - ptr::copy_nonoverlapping(from, dstp, 8); - ptr::copy_nonoverlapping( - from.add(8), - dstp.add(8), - 8, - ); - - // Advance input: literal skips len data bytes, - // copy skips tag_type trailing bytes. - // This branch is the only tag-type-dependent operation - // AFTER the memory copy, minimizing misprediction cost. - if tag_type == 0 { - self.s = old_s + len; - } else { - self.s = old_s + tag_type; - } - self.d += len; - } - } - - // Slow path: handle one tag, then re-enter the fast loop. - if self.s >= src_len { - break 'outer; - } + while self.s < self.src.len() { let byte = self.src[self.s]; self.s += 1; if byte & 0b000000_11 == 0 { @@ -285,9 +193,9 @@ impl<'s, 'd> Decompress<'s, 'd> { self.read_copy(byte)?; } } - if self.d != dst_len { + if self.d != self.dst.len() { return Err(Error::HeaderMismatch { - expected_len: dst_len as u64, + expected_len: self.dst.len() as u64, got_len: self.d as u64, }); } From eb89f9a15bd56042ec9ca40e8f3908e9918b185f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 11:50:51 +0100 Subject: [PATCH 03/41] Optimize decompression --- src/decompress.rs | 90 ++++++++++------------------------------------- 1 file changed, 18 insertions(+), 72 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 0714dc7..6d676cc 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -9,68 +9,6 @@ use crate::MAX_INPUT_SIZE; /// tag byte. const TAG_LOOKUP_TABLE: TagLookupTable = TagLookupTable(tag::TAG_LOOKUP_TABLE); -/// Combined length-and-offset table inspired by C++ snappy's kLengthMinusOffset. -/// -/// For each tag byte, encodes (as i16): -/// - Short literals (len 1-16): `length - 256` (always negative) -/// - Copy-1: `length - (offset_high_bits << 8)` -/// - Copy-2 (len 1-16): `length` -/// - Extended literals, long copies, copy-4: flag value (0x0180) -/// -/// The key property: for common tags (short literals and copies where -/// offset > length), `entry <= ExtractOffset(trailing_bytes, tag_type)`, -/// allowing a single unified code path without branching on tag type. -const LENGTH_MINUS_OFFSET: [i16; 256] = make_length_minus_offset(); - -const fn make_length_minus_offset() -> [i16; 256] { - let mut table = [0i16; 256]; - let mut i = 0usize; - while i < 256 { - let tag = i as u8; - let tag_type = tag & 3; - if tag_type == 0 { - // Literal - let lit_len = (tag >> 2) + 1; - if lit_len <= 16 { - // Common short literal: entry = len - 256 (negative) - table[i] = (lit_len as i16) - 256; - } else { - // Long/extended literal: exit to slow path - table[i] = 0x0180u16 as i16; - } - } else if tag_type == 1 { - // Copy-1: len 4-11, offset has high bits in tag - let len = (4 + ((tag >> 2) & 7)) as i16; - let offset_high = ((tag >> 5) & 7) as i16; - table[i] = len - (offset_high << 8); - } else if tag_type == 2 { - // Copy-2 - let len = (1 + (tag >> 2)) as i16; - if len <= 16 { - table[i] = len; - } else { - // Long copy-2: exit to slow path - table[i] = 0x0180u16 as i16; - } - } else { - // Copy-4: exit to slow path - table[i] = 0x0180u16 as i16; - } - i += 1; - } - table -} - -/// Extract copy offset from trailing bytes using a packed 64-bit mask constant. -/// Returns 0 for literals, low byte for copy-1, full u16 for copy-2, 0 for copy-4. -#[inline(always)] -fn extract_offset(val: u32, tag_type: usize) -> u32 { - const MASKS: u64 = 0x0000FFFF00FF0000u64; - let mask = ((MASKS >> (tag_type * 16)) & 0xFFFF) as u32; - val & mask -} - - /// Returns the decompressed size (in bytes) of the compressed bytes given. /// /// `input` must be a sequence of bytes returned by a conforming Snappy @@ -290,9 +228,16 @@ impl<'s, 'd> Decompress<'s, 'd> { // Find the copy offset and len, then advance the input past the copy. // The rest of this function deals with reading/writing to output only. let entry = TAG_LOOKUP_TABLE.entry(tag_byte); - let offset = entry.offset(self.src, self.s)?; + // Compute num_tag_bytes from tag_type directly (1→1, 2→2, 3→4) + // instead of from entry.num_tag_bytes(). This is available as soon + // as the tag byte is loaded, breaking the serial dependency: + // tag → table_load → extract_num_tag_bytes → compute_mask. + let tag_type = (tag_byte & 3) as usize; + // tag_type: 1→1, 2→2, 3→4 trailing bytes + let num_tag_bytes = tag_type + (tag_type == 3) as usize; + let offset = entry.offset_with_ntb(self.src, self.s, num_tag_bytes)?; let len = entry.len(); - self.s += entry.num_tag_bytes(); + self.s += num_tag_bytes; // What we really care about here is whether `d == 0` or `d < offset`. // To save an extra branch, use `d < offset - 1` instead. If `d` is @@ -426,12 +371,6 @@ impl TagLookupTable { struct TagEntry(usize); impl TagEntry { - /// Return the total number of bytes proceding this tag byte required to - /// encode the offset. - fn num_tag_bytes(&self) -> usize { - self.0 >> 11 - } - /// Return the total copy length, capped at 255. fn len(&self) -> usize { self.0 & 0xFF @@ -443,8 +382,15 @@ impl TagEntry { /// /// This requires reading from the compressed input since the offset is /// encoded in bytes proceding the tag byte. - fn offset(&self, src: &[u8], s: usize) -> Result { - let num_tag_bytes = self.num_tag_bytes(); + /// + /// `num_tag_bytes` is passed in directly (computed from `tag & 3`) + /// to break the serial dependency through the table entry. + fn offset_with_ntb( + &self, + src: &[u8], + s: usize, + num_tag_bytes: usize, + ) -> Result { let trailer = // It is critical for this case to come first, since it is the // fast path. We really hope that this case gets branch From 6896a67020d9e8a0483a82d8fc3d5772bde241db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 11:59:08 +0100 Subject: [PATCH 04/41] Optimize decompression --- src/decompress.rs | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/decompress.rs b/src/decompress.rs index 6d676cc..034b409 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -256,14 +256,52 @@ impl<'s, 'd> Decompress<'s, 'd> { // loads/stores. if offset >= 8 && len <= 16 && self.d + 16 <= self.dst.len() { unsafe { + // SAFETY: We know dstp points to at least 16 bytes of memory + // from the condition above, and we also know that dstp is + // preceded by at least `offset` bytes from the `d <= offset` + // check above. + // + // We also know that dstp and dstp-8 do not overlap from the + // check above, justifying the use of copy_nonoverlapping. let dstp = self.dst.as_mut_ptr().add(self.d); let srcp = dstp.sub(offset); + // We can't do a single 16 byte load/store because src/dst may + // overlap with each other. Namely, the second copy here may + // copy bytes written in the first copy! ptr::copy_nonoverlapping(srcp, dstp, 8); ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); } + // If we have some wiggle room, try to decompress the copy 16 bytes + // at a time with 128 bit unaligned loads/stores. Remember, we can't + // just do a memcpy because decompressing copies may require copying + // overlapping memory. + // + // We need the extra wiggle room to make effective use of 128 bit + // loads/stores. Even if the store ends up copying more data than we + // need, we're careful to advance `d` by the correct amount at the end. } else if end + 24 <= self.dst.len() { unsafe { + // SAFETY: We know that dstp is preceded by at least `offset` + // bytes from the `d <= offset` check above. + // + // We don't know whether dstp overlaps with srcp, so we start + // by copying from srcp to dstp until they no longer overlap. + // The worst case is when dstp-src = 3 and copy length = 1. The + // first loop will issue these copy operations before stopping: + // + // [-1, 14] -> [0, 15] + // [-1, 14] -> [3, 18] + // [-1, 14] -> [9, 24] + // + // But the copy had length 1, so it was only supposed to write + // to [0, 0]. But the last copy wrote to [9, 24], which is 24 + // extra bytes in dst *beyond* the end of the copy, which is + // guaranteed by the conditional above. + + // Save destination length here to avoid a reborrow UB violation + // under the Tree Borrows model. let dest_len = self.dst.len(); + let mut dstp = self.dst.as_mut_ptr().add(self.d); let mut srcp = dstp.sub(offset); loop { @@ -272,6 +310,7 @@ impl<'s, 'd> Decompress<'s, 'd> { if diff >= 16 { break; } + // srcp and dstp can overlap, so use ptr::copy. debug_assert!(self.d + 16 <= dest_len); ptr::copy(srcp, dstp, 16); self.d += diff as usize; @@ -283,6 +322,8 @@ impl<'s, 'd> Decompress<'s, 'd> { dstp = dstp.add(16); self.d += 16; } + // At this point, `d` is likely wrong. We correct it before + // returning. It's correct value is `end`. } } else { if end > self.dst.len() { @@ -291,6 +332,8 @@ impl<'s, 'd> Decompress<'s, 'd> { dst_len: (self.dst.len() - self.d) as u64, }); } + // Finally, the slow byte-by-byte case, which should only be used + // for the last few bytes of decompression. while self.d != end { self.dst[self.d] = self.dst[self.d - offset]; self.d += 1; From fec76ff0d70e504f716b5bea5bfcf139803934f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 12:47:38 +0100 Subject: [PATCH 05/41] Optimize decompression --- src/decompress.rs | 96 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/src/decompress.rs b/src/decompress.rs index 034b409..9538c02 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -121,6 +121,14 @@ impl<'s, 'd> Decompress<'s, 'd> { /// This assumes that the header has already been read and that `dst` is /// big enough to store all decompressed bytes. fn decompress(&mut self) -> Result<()> { + // Fast loop using raw pointers to minimize per-tag overhead. + // We stay in this loop as long as there's enough headroom in both + // src and dst for the worst-case fast-path tag (1 tag + 4 trailer + // + 16 data = 21 src bytes, 16 dst bytes). + unsafe { + self.decompress_fast()?; + } + // Slow loop for the remaining bytes near the end of the buffers. while self.s < self.src.len() { let byte = self.src[self.s]; self.s += 1; @@ -140,6 +148,94 @@ impl<'s, 'd> Decompress<'s, 'd> { Ok(()) } + /// Fast decompression loop using raw pointers. + /// + /// Handles common cases (small literals, copies with offset >= 8 and + /// len <= 16) with minimal per-tag overhead. For uncommon cases, calls + /// the original methods and continues the fast loop. + #[inline(always)] + unsafe fn decompress_fast(&mut self) -> Result<()> { + let src = self.src.as_ptr(); + let dst = self.dst.as_mut_ptr(); + let src_len = self.src.len(); + let dst_len = self.dst.len(); + + while self.s + 5 <= src_len { + let byte = *src.add(self.s); + + if byte & 3 == 0 { + // Literal tag + let len = (byte >> 2) as usize + 1; + if len <= 16 + && self.s + 1 + 16 <= src_len + && self.d + 16 <= dst_len + { + self.s += 1; + ptr::copy_nonoverlapping( + src.add(self.s), + dst.add(self.d), + 16, + ); + self.s += len; + self.d += len; + continue; + } + // Slow path for extended literals or near boundaries + self.s += 1; + self.read_literal(len)?; + continue; + } + + // Copy tag: compute offset/len using raw pointers + let entry_val = TAG_LOOKUP_TABLE.0[byte as usize] as usize; + let tag_type = (byte & 3) as usize; + let num_tag_bytes = tag_type + (tag_type == 3) as usize; + let len = entry_val & 0xFF; + self.s += 1; + + // Check we can read the trailer + if self.s + 4 > src_len { + // Near end, fall back to safe read_copy + self.s -= 1; + self.s += 1; + self.read_copy(byte)?; + continue; + } + + // Fast trailer load + let mask = u32::MAX >> ((4 - num_tag_bytes as u32) << 3); + let trailer = + bytes::loadu_u32_le(src.add(self.s)) as usize & mask as usize; + let offset = (entry_val & 0x700) | trailer; + self.s += num_tag_bytes; + + // Validate offset + if self.d <= offset.wrapping_sub(1) { + return Err(Error::Offset { + offset: offset as u64, + dst_pos: self.d as u64, + }); + } + + if offset >= 8 && len <= 16 && self.d + 16 <= dst_len { + // Fast copy: two non-overlapping 8-byte copies + let dstp = dst.add(self.d); + let srcp = dstp.sub(offset); + ptr::copy_nonoverlapping(srcp, dstp, 8); + ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); + self.d += len; + continue; + } + + // Slow path for complex copies (small offset, large len, + // or near dst boundary). Reconstruct state for read_copy. + self.s -= num_tag_bytes + 1; + self.s += 1; + self.read_copy(byte)?; + } + Ok(()) + } + /// Decompresses a literal from `src` starting at `s` to `dst` starting at /// `d` and returns the updated values of `s` and `d`. `s` should point to /// the byte immediately proceding the literal tag byte. From 05ab2961751cbe771309927a9974d3983fa98a1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 12:49:39 +0100 Subject: [PATCH 06/41] Optimize decompression --- src/decompress.rs | 53 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 9538c02..2cc0a87 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -150,9 +150,9 @@ impl<'s, 'd> Decompress<'s, 'd> { /// Fast decompression loop using raw pointers. /// - /// Handles common cases (small literals, copies with offset >= 8 and - /// len <= 16) with minimal per-tag overhead. For uncommon cases, calls - /// the original methods and continues the fast loop. + /// Handles common cases with minimal per-tag overhead. For uncommon + /// cases (extended literals, copy-4, near boundaries), calls the + /// original methods and continues. #[inline(always)] unsafe fn decompress_fast(&mut self) -> Result<()> { let src = self.src.as_ptr(); @@ -195,7 +195,6 @@ impl<'s, 'd> Decompress<'s, 'd> { // Check we can read the trailer if self.s + 4 > src_len { - // Near end, fall back to safe read_copy self.s -= 1; self.s += 1; self.read_copy(byte)?; @@ -217,21 +216,55 @@ impl<'s, 'd> Decompress<'s, 'd> { }); } + let end = self.d + len; + if offset >= 8 && len <= 16 && self.d + 16 <= dst_len { // Fast copy: two non-overlapping 8-byte copies let dstp = dst.add(self.d); let srcp = dstp.sub(offset); ptr::copy_nonoverlapping(srcp, dstp, 8); ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); - self.d += len; + self.d = end; continue; } - // Slow path for complex copies (small offset, large len, - // or near dst boundary). Reconstruct state for read_copy. - self.s -= num_tag_bytes + 1; - self.s += 1; - self.read_copy(byte)?; + // Handle remaining copy cases inline to avoid re-computing + // offset/len in read_copy. + if end + 24 <= dst_len { + let mut dstp = dst.add(self.d); + let mut srcp = dstp.sub(offset); + // First expand overlapping region to >= 16 bytes + loop { + let diff = (dstp as usize) - (srcp as usize); + if diff >= 16 { + break; + } + ptr::copy(srcp, dstp, 16); + self.d += diff; + dstp = dstp.add(diff); + } + // Then copy 16 at a time (non-overlapping) + while self.d < end { + ptr::copy_nonoverlapping(srcp, dstp, 16); + srcp = srcp.add(16); + dstp = dstp.add(16); + self.d += 16; + } + self.d = end; + continue; + } + + // Near dst boundary: byte-by-byte + if end > dst_len { + return Err(Error::CopyWrite { + len: len as u64, + dst_len: (dst_len - self.d) as u64, + }); + } + while self.d != end { + *dst.add(self.d) = *dst.add(self.d - offset); + self.d += 1; + } } Ok(()) } From 4c2856b41db0971eaf5834ff3fdbe6c9f9a3f022 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 13:06:54 +0100 Subject: [PATCH 07/41] Optimize decompression --- src/decompress.rs | 126 ++++++++++++++++++++-------------------------- 1 file changed, 55 insertions(+), 71 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 2cc0a87..f10ca25 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -150,9 +150,9 @@ impl<'s, 'd> Decompress<'s, 'd> { /// Fast decompression loop using raw pointers. /// - /// Handles common cases with minimal per-tag overhead. For uncommon - /// cases (extended literals, copy-4, near boundaries), calls the - /// original methods and continues. + /// Handles common fast-path tags (small literals, copies with + /// offset >= 8 and len <= 16) with raw pointer ops. Delegates + /// complex cases to the existing safe methods and continues. #[inline(always)] unsafe fn decompress_fast(&mut self) -> Result<()> { let src = self.src.as_ptr(); @@ -180,91 +180,75 @@ impl<'s, 'd> Decompress<'s, 'd> { self.d += len; continue; } - // Slow path for extended literals or near boundaries + // Slow path: extended literals or near boundaries self.s += 1; self.read_literal(len)?; continue; } - // Copy tag: compute offset/len using raw pointers + // Copy tag: fast-path offset/len computation let entry_val = TAG_LOOKUP_TABLE.0[byte as usize] as usize; let tag_type = (byte & 3) as usize; let num_tag_bytes = tag_type + (tag_type == 3) as usize; let len = entry_val & 0xFF; + // Save position before consuming tag, for potential backup + let s_tag = self.s; self.s += 1; - // Check we can read the trailer - if self.s + 4 > src_len { - self.s -= 1; - self.s += 1; - self.read_copy(byte)?; - continue; - } - - // Fast trailer load - let mask = u32::MAX >> ((4 - num_tag_bytes as u32) << 3); - let trailer = - bytes::loadu_u32_le(src.add(self.s)) as usize & mask as usize; - let offset = (entry_val & 0x700) | trailer; - self.s += num_tag_bytes; - - // Validate offset - if self.d <= offset.wrapping_sub(1) { - return Err(Error::Offset { - offset: offset as u64, - dst_pos: self.d as u64, - }); - } - - let end = self.d + len; + if self.s + 4 <= src_len { + let mask = u32::MAX >> ((4 - num_tag_bytes as u32) << 3); + let trailer = + bytes::loadu_u32_le(src.add(self.s)) as usize + & mask as usize; + let offset = (entry_val & 0x700) | trailer; + self.s += num_tag_bytes; + + if self.d <= offset.wrapping_sub(1) { + return Err(Error::Offset { + offset: offset as u64, + dst_pos: self.d as u64, + }); + } - if offset >= 8 && len <= 16 && self.d + 16 <= dst_len { - // Fast copy: two non-overlapping 8-byte copies - let dstp = dst.add(self.d); - let srcp = dstp.sub(offset); - ptr::copy_nonoverlapping(srcp, dstp, 8); - ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); - self.d = end; - continue; - } + if offset >= 8 && len <= 16 && self.d + 16 <= dst_len { + let dstp = dst.add(self.d); + let srcp = dstp.sub(offset); + ptr::copy_nonoverlapping(srcp, dstp, 8); + ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); + self.d += len; + continue; + } - // Handle remaining copy cases inline to avoid re-computing - // offset/len in read_copy. - if end + 24 <= dst_len { - let mut dstp = dst.add(self.d); - let mut srcp = dstp.sub(offset); - // First expand overlapping region to >= 16 bytes - loop { - let diff = (dstp as usize) - (srcp as usize); - if diff >= 16 { - break; + // Medium path: long copies with offset >= 8 + let end = self.d + len; + if offset >= 8 && end + 24 <= dst_len { + let mut dstp = dst.add(self.d); + let mut srcp = dstp.sub(offset); + // Expand overlap until gap >= 16 + loop { + let diff = (dstp as usize) - (srcp as usize); + if diff >= 16 { + break; + } + ptr::copy(srcp, dstp, 16); + self.d += diff; + dstp = dstp.add(diff); } - ptr::copy(srcp, dstp, 16); - self.d += diff; - dstp = dstp.add(diff); - } - // Then copy 16 at a time (non-overlapping) - while self.d < end { - ptr::copy_nonoverlapping(srcp, dstp, 16); - srcp = srcp.add(16); - dstp = dstp.add(16); - self.d += 16; + // Non-overlapping 16-byte copies + while self.d < end { + ptr::copy_nonoverlapping(srcp, dstp, 16); + srcp = srcp.add(16); + dstp = dstp.add(16); + self.d += 16; + } + self.d = end; + continue; } - self.d = end; - continue; } - // Near dst boundary: byte-by-byte - if end > dst_len { - return Err(Error::CopyWrite { - len: len as u64, - dst_len: (dst_len - self.d) as u64, - }); - } - while self.d != end { - *dst.add(self.d) = *dst.add(self.d - offset); - self.d += 1; - } + // Slow path: restore s to just past tag byte, delegate + self.s = s_tag + 1; + self.read_copy(byte)?; } Ok(()) } From 3aa53c47b773bfa52028c689576f093afb0521c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 15:09:52 +0100 Subject: [PATCH 08/41] Optimize decompression --- src/decompress.rs | 88 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 75 insertions(+), 13 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index f10ca25..d395a65 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -180,6 +180,36 @@ impl<'s, 'd> Decompress<'s, 'd> { self.d += len; continue; } + // Medium-fast path for literals with len 17-60: + // unrolled 16-byte copies, avoiding the overhead of + // read_literal's bounds checks and extended-length handling. + if len <= 60 + && self.s + 1 + len + 16 <= src_len + && self.d + len + 16 <= dst_len + { + self.s += 1; + let srcp = src.add(self.s); + let dstp = dst.add(self.d); + ptr::copy_nonoverlapping(srcp, dstp, 16); + ptr::copy_nonoverlapping(srcp.add(16), dstp.add(16), 16); + if len > 32 { + ptr::copy_nonoverlapping( + srcp.add(32), + dstp.add(32), + 16, + ); + if len > 48 { + ptr::copy_nonoverlapping( + srcp.add(48), + dstp.add(48), + 16, + ); + } + } + self.s += len; + self.d += len; + continue; + } // Slow path: extended literals or near boundaries self.s += 1; self.read_literal(len)?; @@ -197,9 +227,8 @@ impl<'s, 'd> Decompress<'s, 'd> { if self.s + 4 <= src_len { let mask = u32::MAX >> ((4 - num_tag_bytes as u32) << 3); - let trailer = - bytes::loadu_u32_le(src.add(self.s)) as usize - & mask as usize; + let trailer = bytes::loadu_u32_le(src.add(self.s)) as usize + & mask as usize; let offset = (entry_val & 0x700) | trailer; self.s += num_tag_bytes; @@ -219,6 +248,32 @@ impl<'s, 'd> Decompress<'s, 'd> { continue; } + // Fast path for medium copies (len 17-64, offset >= 16). + // Two 16-byte non-overlapping copies handle up to 32 bytes. + // Four handle up to 64 (max copy-2 length). + if offset >= 16 && self.d + len + 16 <= dst_len { + let dstp = dst.add(self.d); + let srcp = dstp.sub(offset); + ptr::copy_nonoverlapping(srcp, dstp, 16); + ptr::copy_nonoverlapping(srcp.add(16), dstp.add(16), 16); + if len > 32 { + ptr::copy_nonoverlapping( + srcp.add(32), + dstp.add(32), + 16, + ); + if len > 48 { + ptr::copy_nonoverlapping( + srcp.add(48), + dstp.add(48), + 16, + ); + } + } + self.d += len; + continue; + } + // Medium path: long copies with offset >= 8 let end = self.d + len; if offset >= 8 && end + 24 <= dst_len { @@ -369,21 +424,28 @@ impl<'s, 'd> Decompress<'s, 'd> { // loads/stores. if offset >= 8 && len <= 16 && self.d + 16 <= self.dst.len() { unsafe { - // SAFETY: We know dstp points to at least 16 bytes of memory - // from the condition above, and we also know that dstp is - // preceded by at least `offset` bytes from the `d <= offset` - // check above. - // - // We also know that dstp and dstp-8 do not overlap from the - // check above, justifying the use of copy_nonoverlapping. let dstp = self.dst.as_mut_ptr().add(self.d); let srcp = dstp.sub(offset); - // We can't do a single 16 byte load/store because src/dst may - // overlap with each other. Namely, the second copy here may - // copy bytes written in the first copy! ptr::copy_nonoverlapping(srcp, dstp, 8); ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); } + } else if offset >= 16 && end + 16 <= self.dst.len() { + unsafe { + let dstp = self.dst.as_mut_ptr().add(self.d); + let srcp = dstp.sub(offset); + ptr::copy_nonoverlapping(srcp, dstp, 16); + ptr::copy_nonoverlapping(srcp.add(16), dstp.add(16), 16); + if len > 32 { + ptr::copy_nonoverlapping(srcp.add(32), dstp.add(32), 16); + if len > 48 { + ptr::copy_nonoverlapping( + srcp.add(48), + dstp.add(48), + 16, + ); + } + } + } // If we have some wiggle room, try to decompress the copy 16 bytes // at a time with 128 bit unaligned loads/stores. Remember, we can't // just do a memcpy because decompressing copies may require copying From d7bc799b9a9dcfd8601a8c20f2ce3c9514956a3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 15:15:52 +0100 Subject: [PATCH 09/41] Optimize decompression --- src/decompress.rs | 109 +++++++++++----------------------------------- 1 file changed, 25 insertions(+), 84 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index d395a65..38d7aeb 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -9,6 +9,22 @@ use crate::MAX_INPUT_SIZE; /// tag byte. const TAG_LOOKUP_TABLE: TagLookupTable = TagLookupTable(tag::TAG_LOOKUP_TABLE); +/// Copy up to 64 bytes using unrolled 16-byte copies. +/// `src` and `dst` must not overlap and must have at least `len + 16` bytes +/// of addressable memory (we always copy in 16-byte chunks). +#[inline(always)] +unsafe fn wide_copy(src: *const u8, dst: *mut u8, len: usize) { + debug_assert!(len <= 64); + ptr::copy_nonoverlapping(src, dst, 16); + ptr::copy_nonoverlapping(src.add(16), dst.add(16), 16); + if len > 32 { + ptr::copy_nonoverlapping(src.add(32), dst.add(32), 16); + if len > 48 { + ptr::copy_nonoverlapping(src.add(48), dst.add(48), 16); + } + } +} + /// Returns the decompressed size (in bytes) of the compressed bytes given. /// /// `input` must be a sequence of bytes returned by a conforming Snappy @@ -121,14 +137,9 @@ impl<'s, 'd> Decompress<'s, 'd> { /// This assumes that the header has already been read and that `dst` is /// big enough to store all decompressed bytes. fn decompress(&mut self) -> Result<()> { - // Fast loop using raw pointers to minimize per-tag overhead. - // We stay in this loop as long as there's enough headroom in both - // src and dst for the worst-case fast-path tag (1 tag + 4 trailer - // + 16 data = 21 src bytes, 16 dst bytes). unsafe { self.decompress_fast()?; } - // Slow loop for the remaining bytes near the end of the buffers. while self.s < self.src.len() { let byte = self.src[self.s]; self.s += 1; @@ -148,11 +159,7 @@ impl<'s, 'd> Decompress<'s, 'd> { Ok(()) } - /// Fast decompression loop using raw pointers. - /// - /// Handles common fast-path tags (small literals, copies with - /// offset >= 8 and len <= 16) with raw pointer ops. Delegates - /// complex cases to the existing safe methods and continues. + /// Fast decompression loop using raw pointers for common cases. #[inline(always)] unsafe fn decompress_fast(&mut self) -> Result<()> { let src = self.src.as_ptr(); @@ -164,7 +171,6 @@ impl<'s, 'd> Decompress<'s, 'd> { let byte = *src.add(self.s); if byte & 3 == 0 { - // Literal tag let len = (byte >> 2) as usize + 1; if len <= 16 && self.s + 1 + 16 <= src_len @@ -180,48 +186,26 @@ impl<'s, 'd> Decompress<'s, 'd> { self.d += len; continue; } - // Medium-fast path for literals with len 17-60: - // unrolled 16-byte copies, avoiding the overhead of - // read_literal's bounds checks and extended-length handling. + // Medium-fast path for literals len 17-60. if len <= 60 && self.s + 1 + len + 16 <= src_len && self.d + len + 16 <= dst_len { self.s += 1; - let srcp = src.add(self.s); - let dstp = dst.add(self.d); - ptr::copy_nonoverlapping(srcp, dstp, 16); - ptr::copy_nonoverlapping(srcp.add(16), dstp.add(16), 16); - if len > 32 { - ptr::copy_nonoverlapping( - srcp.add(32), - dstp.add(32), - 16, - ); - if len > 48 { - ptr::copy_nonoverlapping( - srcp.add(48), - dstp.add(48), - 16, - ); - } - } + wide_copy(src.add(self.s), dst.add(self.d), len); self.s += len; self.d += len; continue; } - // Slow path: extended literals or near boundaries self.s += 1; self.read_literal(len)?; continue; } - // Copy tag: fast-path offset/len computation let entry_val = TAG_LOOKUP_TABLE.0[byte as usize] as usize; let tag_type = (byte & 3) as usize; let num_tag_bytes = tag_type + (tag_type == 3) as usize; let len = entry_val & 0xFF; - // Save position before consuming tag, for potential backup let s_tag = self.s; self.s += 1; @@ -248,38 +232,19 @@ impl<'s, 'd> Decompress<'s, 'd> { continue; } - // Fast path for medium copies (len 17-64, offset >= 16). - // Two 16-byte non-overlapping copies handle up to 32 bytes. - // Four handle up to 64 (max copy-2 length). + // Medium copies (len 17-64, offset >= 16): no overlap. if offset >= 16 && self.d + len + 16 <= dst_len { let dstp = dst.add(self.d); - let srcp = dstp.sub(offset); - ptr::copy_nonoverlapping(srcp, dstp, 16); - ptr::copy_nonoverlapping(srcp.add(16), dstp.add(16), 16); - if len > 32 { - ptr::copy_nonoverlapping( - srcp.add(32), - dstp.add(32), - 16, - ); - if len > 48 { - ptr::copy_nonoverlapping( - srcp.add(48), - dstp.add(48), - 16, - ); - } - } + wide_copy(dstp.sub(offset), dstp, len); self.d += len; continue; } - // Medium path: long copies with offset >= 8 + // Long copies with offset >= 8 let end = self.d + len; if offset >= 8 && end + 24 <= dst_len { let mut dstp = dst.add(self.d); let mut srcp = dstp.sub(offset); - // Expand overlap until gap >= 16 loop { let diff = (dstp as usize) - (srcp as usize); if diff >= 16 { @@ -289,7 +254,6 @@ impl<'s, 'd> Decompress<'s, 'd> { self.d += diff; dstp = dstp.add(diff); } - // Non-overlapping 16-byte copies while self.d < end { ptr::copy_nonoverlapping(srcp, dstp, 16); srcp = srcp.add(16); @@ -301,7 +265,6 @@ impl<'s, 'd> Decompress<'s, 'd> { } } - // Slow path: restore s to just past tag byte, delegate self.s = s_tag + 1; self.read_copy(byte)?; } @@ -396,12 +359,9 @@ impl<'s, 'd> Decompress<'s, 'd> { // Find the copy offset and len, then advance the input past the copy. // The rest of this function deals with reading/writing to output only. let entry = TAG_LOOKUP_TABLE.entry(tag_byte); - // Compute num_tag_bytes from tag_type directly (1→1, 2→2, 3→4) - // instead of from entry.num_tag_bytes(). This is available as soon - // as the tag byte is loaded, breaking the serial dependency: - // tag → table_load → extract_num_tag_bytes → compute_mask. + // Compute num_tag_bytes from tag_type directly to break serial + // dependency through the table entry. let tag_type = (tag_byte & 3) as usize; - // tag_type: 1→1, 2→2, 3→4 trailing bytes let num_tag_bytes = tag_type + (tag_type == 3) as usize; let offset = entry.offset_with_ntb(self.src, self.s, num_tag_bytes)?; let len = entry.len(); @@ -432,19 +392,7 @@ impl<'s, 'd> Decompress<'s, 'd> { } else if offset >= 16 && end + 16 <= self.dst.len() { unsafe { let dstp = self.dst.as_mut_ptr().add(self.d); - let srcp = dstp.sub(offset); - ptr::copy_nonoverlapping(srcp, dstp, 16); - ptr::copy_nonoverlapping(srcp.add(16), dstp.add(16), 16); - if len > 32 { - ptr::copy_nonoverlapping(srcp.add(32), dstp.add(32), 16); - if len > 48 { - ptr::copy_nonoverlapping( - srcp.add(48), - dstp.add(48), - 16, - ); - } - } + wide_copy(dstp.sub(offset), dstp, len); } // If we have some wiggle room, try to decompress the copy 16 bytes // at a time with 128 bit unaligned loads/stores. Remember, we can't @@ -600,9 +548,6 @@ impl TagEntry { /// /// This requires reading from the compressed input since the offset is /// encoded in bytes proceding the tag byte. - /// - /// `num_tag_bytes` is passed in directly (computed from `tag & 3`) - /// to break the serial dependency through the table entry. fn offset_with_ntb( &self, src: &[u8], @@ -618,10 +563,6 @@ impl TagEntry { // SAFETY: The conditional above guarantees that // src[s..s+4] is valid to read from. let p = src.as_ptr().add(s); - // Mask to extract only the bytes we need from the u32. - // num_tag_bytes is 1, 2, or 4 for copy tags, so the - // shift is always 24, 16, or 0 (all < 32). - // This avoids a table lookup on the critical path. let mask = u32::MAX >> ((4 - num_tag_bytes as u32) << 3); bytes::loadu_u32_le(p) as usize & mask as usize } From 2a5a1c8c8807b5ef4307a23afe7746961bc8dd9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 15:31:12 +0100 Subject: [PATCH 10/41] Optimize decompression --- src/decompress.rs | 88 +++++++++++++++-------------------------------- 1 file changed, 28 insertions(+), 60 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 38d7aeb..517f272 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -25,6 +25,32 @@ unsafe fn wide_copy(src: *const u8, dst: *mut u8, len: usize) { } } +/// Copy `len` bytes from `dst - offset` into `dst`, handling overlapping +/// regions by expanding with `ptr::copy` until the gap >= 16, then +/// switching to non-overlapping 16-byte chunks. +/// +/// Caller must ensure `dst + len + 24` is writable and that `dst` is +/// preceded by at least `offset` valid bytes. +#[inline(always)] +unsafe fn overlapping_copy(dst: *mut u8, offset: usize, len: usize) { + let end = dst.add(len); + let mut dstp = dst; + let mut srcp = dst.sub(offset); + loop { + let diff = (dstp as usize) - (srcp as usize); + if diff >= 16 { + break; + } + ptr::copy(srcp, dstp, 16); + dstp = dstp.add(diff); + } + while dstp < end { + ptr::copy_nonoverlapping(srcp, dstp, 16); + srcp = srcp.add(16); + dstp = dstp.add(16); + } +} + /// Returns the decompressed size (in bytes) of the compressed bytes given. /// /// `input` must be a sequence of bytes returned by a conforming Snappy @@ -243,23 +269,7 @@ impl<'s, 'd> Decompress<'s, 'd> { // Long copies with offset >= 8 let end = self.d + len; if offset >= 8 && end + 24 <= dst_len { - let mut dstp = dst.add(self.d); - let mut srcp = dstp.sub(offset); - loop { - let diff = (dstp as usize) - (srcp as usize); - if diff >= 16 { - break; - } - ptr::copy(srcp, dstp, 16); - self.d += diff; - dstp = dstp.add(diff); - } - while self.d < end { - ptr::copy_nonoverlapping(srcp, dstp, 16); - srcp = srcp.add(16); - dstp = dstp.add(16); - self.d += 16; - } + overlapping_copy(dst.add(self.d), offset, len); self.d = end; continue; } @@ -404,49 +414,7 @@ impl<'s, 'd> Decompress<'s, 'd> { // need, we're careful to advance `d` by the correct amount at the end. } else if end + 24 <= self.dst.len() { unsafe { - // SAFETY: We know that dstp is preceded by at least `offset` - // bytes from the `d <= offset` check above. - // - // We don't know whether dstp overlaps with srcp, so we start - // by copying from srcp to dstp until they no longer overlap. - // The worst case is when dstp-src = 3 and copy length = 1. The - // first loop will issue these copy operations before stopping: - // - // [-1, 14] -> [0, 15] - // [-1, 14] -> [3, 18] - // [-1, 14] -> [9, 24] - // - // But the copy had length 1, so it was only supposed to write - // to [0, 0]. But the last copy wrote to [9, 24], which is 24 - // extra bytes in dst *beyond* the end of the copy, which is - // guaranteed by the conditional above. - - // Save destination length here to avoid a reborrow UB violation - // under the Tree Borrows model. - let dest_len = self.dst.len(); - - let mut dstp = self.dst.as_mut_ptr().add(self.d); - let mut srcp = dstp.sub(offset); - loop { - debug_assert!(dstp >= srcp); - let diff = (dstp as usize) - (srcp as usize); - if diff >= 16 { - break; - } - // srcp and dstp can overlap, so use ptr::copy. - debug_assert!(self.d + 16 <= dest_len); - ptr::copy(srcp, dstp, 16); - self.d += diff as usize; - dstp = dstp.add(diff); - } - while self.d < end { - ptr::copy_nonoverlapping(srcp, dstp, 16); - srcp = srcp.add(16); - dstp = dstp.add(16); - self.d += 16; - } - // At this point, `d` is likely wrong. We correct it before - // returning. It's correct value is `end`. + overlapping_copy(self.dst.as_mut_ptr().add(self.d), offset, len); } } else { if end > self.dst.len() { From 7f0c2b18950907f46bac63ef2d5be54f6ac93a2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 15:39:03 +0100 Subject: [PATCH 11/41] Optimize decompression --- src/decompress.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 517f272..dc9034a 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -266,9 +266,8 @@ impl<'s, 'd> Decompress<'s, 'd> { continue; } - // Long copies with offset >= 8 let end = self.d + len; - if offset >= 8 && end + 24 <= dst_len { + if end + 24 <= dst_len { overlapping_copy(dst.add(self.d), offset, len); self.d = end; continue; From f3d0ef1e565952c4b5d1dba2e41277c38ef86a1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 19:25:41 +0100 Subject: [PATCH 12/41] Snappy tweaks --- src/decompress.rs | 67 ++++++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index dc9034a..1764646 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -235,43 +235,44 @@ impl<'s, 'd> Decompress<'s, 'd> { let s_tag = self.s; self.s += 1; - if self.s + 4 <= src_len { - let mask = u32::MAX >> ((4 - num_tag_bytes as u32) << 3); - let trailer = bytes::loadu_u32_le(src.add(self.s)) as usize - & mask as usize; - let offset = (entry_val & 0x700) | trailer; - self.s += num_tag_bytes; + // Safety: The loop condition `self.s + 5 <= src_len` (checked + // before self.s was modified) guarantees that at least 4 bytes + // are available at src + self.s for the offset read. + let mask = u32::MAX >> ((4 - num_tag_bytes as u32) << 3); + let trailer = bytes::loadu_u32_le(src.add(self.s)) as usize + & mask as usize; + let offset = (entry_val & 0x700) | trailer; + self.s += num_tag_bytes; - if self.d <= offset.wrapping_sub(1) { - return Err(Error::Offset { - offset: offset as u64, - dst_pos: self.d as u64, - }); - } + if self.d <= offset.wrapping_sub(1) { + return Err(Error::Offset { + offset: offset as u64, + dst_pos: self.d as u64, + }); + } - if offset >= 8 && len <= 16 && self.d + 16 <= dst_len { - let dstp = dst.add(self.d); - let srcp = dstp.sub(offset); - ptr::copy_nonoverlapping(srcp, dstp, 8); - ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); - self.d += len; - continue; - } + if offset >= 8 && len <= 16 && self.d + 16 <= dst_len { + let dstp = dst.add(self.d); + let srcp = dstp.sub(offset); + ptr::copy_nonoverlapping(srcp, dstp, 8); + ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); + self.d += len; + continue; + } - // Medium copies (len 17-64, offset >= 16): no overlap. - if offset >= 16 && self.d + len + 16 <= dst_len { - let dstp = dst.add(self.d); - wide_copy(dstp.sub(offset), dstp, len); - self.d += len; - continue; - } + // Medium copies (len 17-64, offset >= 16): no overlap. + if offset >= 16 && self.d + len + 16 <= dst_len { + let dstp = dst.add(self.d); + wide_copy(dstp.sub(offset), dstp, len); + self.d += len; + continue; + } - let end = self.d + len; - if end + 24 <= dst_len { - overlapping_copy(dst.add(self.d), offset, len); - self.d = end; - continue; - } + let end = self.d + len; + if end + 24 <= dst_len { + overlapping_copy(dst.add(self.d), offset, len); + self.d = end; + continue; } self.s = s_tag + 1; From 2cb1e53ad93e770654bceaa45e443a90080a92ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 19:49:15 +0100 Subject: [PATCH 13/41] Snappy tweaks --- src/decompress.rs | 56 ++++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 1764646..9706f62 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -186,6 +186,13 @@ impl<'s, 'd> Decompress<'s, 'd> { } /// Fast decompression loop using raw pointers for common cases. + /// + /// The loop condition guarantees sufficient headroom in both source and + /// destination buffers to eliminate most bounds checks from the loop body: + /// - `s + 17 <= src_len`: ensures 16 bytes of literal data + 1 tag byte + /// can always be read, and 4 bytes of copy offset data (since 17 > 5). + /// - `d + 88 <= dst_len`: ensures max copy (64 bytes) + overlapping_copy + /// wiggle room (24 bytes) always fits, so no destination checks needed. #[inline(always)] unsafe fn decompress_fast(&mut self) -> Result<()> { let src = self.src.as_ptr(); @@ -193,15 +200,14 @@ impl<'s, 'd> Decompress<'s, 'd> { let src_len = self.src.len(); let dst_len = self.dst.len(); - while self.s + 5 <= src_len { + while self.s + 17 <= src_len && self.d + 88 <= dst_len { let byte = *src.add(self.s); if byte & 3 == 0 { let len = (byte >> 2) as usize + 1; - if len <= 16 - && self.s + 1 + 16 <= src_len - && self.d + 16 <= dst_len - { + if len <= 16 { + // Always safe: loop condition guarantees + // s + 1 + 16 <= src_len and d + 16 <= dst_len. self.s += 1; ptr::copy_nonoverlapping( src.add(self.s), @@ -213,10 +219,9 @@ impl<'s, 'd> Decompress<'s, 'd> { continue; } // Medium-fast path for literals len 17-60. - if len <= 60 - && self.s + 1 + len + 16 <= src_len - && self.d + len + 16 <= dst_len - { + // Destination is guaranteed safe by loop condition + // (d + 76 <= d + 88 <= dst_len). Source needs checking. + if len <= 60 && self.s + 1 + len + 16 <= src_len { self.s += 1; wide_copy(src.add(self.s), dst.add(self.d), len); self.s += len; @@ -232,12 +237,10 @@ impl<'s, 'd> Decompress<'s, 'd> { let tag_type = (byte & 3) as usize; let num_tag_bytes = tag_type + (tag_type == 3) as usize; let len = entry_val & 0xFF; - let s_tag = self.s; self.s += 1; - // Safety: The loop condition `self.s + 5 <= src_len` (checked - // before self.s was modified) guarantees that at least 4 bytes - // are available at src + self.s for the offset read. + // Safety: loop condition guarantees s + 17 <= src_len, + // so after s += 1 there are at least 16 >= 4 bytes available. let mask = u32::MAX >> ((4 - num_tag_bytes as u32) << 3); let trailer = bytes::loadu_u32_le(src.add(self.s)) as usize & mask as usize; @@ -251,7 +254,18 @@ impl<'s, 'd> Decompress<'s, 'd> { }); } - if offset >= 8 && len <= 16 && self.d + 16 <= dst_len { + // All destination bounds are guaranteed by the loop condition + // (d + 88 <= dst_len covers max copy 64 + 24 wiggle room). + if len <= 16 && offset >= 16 { + // Single 128-bit non-overlapping copy (compiles to ldr q/str q). + let dstp = dst.add(self.d); + ptr::copy_nonoverlapping(dstp.sub(offset), dstp, 16); + self.d += len; + continue; + } + + if offset >= 8 && len <= 16 { + // For offset 8-15: two 64-bit copies to avoid UB from overlap. let dstp = dst.add(self.d); let srcp = dstp.sub(offset); ptr::copy_nonoverlapping(srcp, dstp, 8); @@ -260,23 +274,15 @@ impl<'s, 'd> Decompress<'s, 'd> { continue; } - // Medium copies (len 17-64, offset >= 16): no overlap. - if offset >= 16 && self.d + len + 16 <= dst_len { + if offset >= 16 { let dstp = dst.add(self.d); wide_copy(dstp.sub(offset), dstp, len); self.d += len; continue; } - let end = self.d + len; - if end + 24 <= dst_len { - overlapping_copy(dst.add(self.d), offset, len); - self.d = end; - continue; - } - - self.s = s_tag + 1; - self.read_copy(byte)?; + overlapping_copy(dst.add(self.d), offset, len); + self.d += len; } Ok(()) } From 36565bd651103b465b5f953c07f8aaae3112ee6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 20:11:06 +0100 Subject: [PATCH 14/41] Improve --- src/decompress.rs | 70 ++++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 9706f62..1b45aaa 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -200,14 +200,20 @@ impl<'s, 'd> Decompress<'s, 'd> { let src_len = self.src.len(); let dst_len = self.dst.len(); - while self.s + 17 <= src_len && self.d + 88 <= dst_len { - let byte = *src.add(self.s); + if !(self.s + 17 <= src_len && self.d + 88 <= dst_len) { + return Ok(()); + } + + // Software-pipelined loop: pre-load the current tag byte and its + // table entry so that after each copy/literal the next iteration's + // loads are already in flight. + let mut byte = *src.add(self.s); + let mut entry_val = TAG_LOOKUP_TABLE.0[byte as usize] as usize; + loop { if byte & 3 == 0 { let len = (byte >> 2) as usize + 1; if len <= 16 { - // Always safe: loop condition guarantees - // s + 1 + 16 <= src_len and d + 16 <= dst_len. self.s += 1; ptr::copy_nonoverlapping( src.add(self.s), @@ -216,31 +222,29 @@ impl<'s, 'd> Decompress<'s, 'd> { ); self.s += len; self.d += len; - continue; - } - // Medium-fast path for literals len 17-60. - // Destination is guaranteed safe by loop condition - // (d + 76 <= d + 88 <= dst_len). Source needs checking. - if len <= 60 && self.s + 1 + len + 16 <= src_len { + } else if len <= 60 && self.s + 1 + len + 16 <= src_len { self.s += 1; wide_copy(src.add(self.s), dst.add(self.d), len); self.s += len; self.d += len; - continue; + } else { + self.s += 1; + self.read_literal(len)?; } - self.s += 1; - self.read_literal(len)?; + // Re-check loop condition, then pre-load next tag. + if !(self.s + 17 <= src_len && self.d + 88 <= dst_len) { + break; + } + byte = *src.add(self.s); + entry_val = TAG_LOOKUP_TABLE.0[byte as usize] as usize; continue; } - let entry_val = TAG_LOOKUP_TABLE.0[byte as usize] as usize; let tag_type = (byte & 3) as usize; let num_tag_bytes = tag_type + (tag_type == 3) as usize; let len = entry_val & 0xFF; self.s += 1; - // Safety: loop condition guarantees s + 17 <= src_len, - // so after s += 1 there are at least 16 >= 4 bytes available. let mask = u32::MAX >> ((4 - num_tag_bytes as u32) << 3); let trailer = bytes::loadu_u32_le(src.add(self.s)) as usize & mask as usize; @@ -254,35 +258,39 @@ impl<'s, 'd> Decompress<'s, 'd> { }); } + // Pre-load next tag byte and table entry BEFORE the copy, + // so the ~4-cycle table lookup overlaps with the copy operation. + // Safety: self.s is valid (we'll re-check the loop condition + // after the copy, but reading one byte past the end is safe + // because src_len >= s + 17 held at loop entry and we advanced + // at most 5 bytes). + byte = *src.add(self.s); + entry_val = TAG_LOOKUP_TABLE.0[byte as usize] as usize; + // All destination bounds are guaranteed by the loop condition // (d + 88 <= dst_len covers max copy 64 + 24 wiggle room). if len <= 16 && offset >= 16 { - // Single 128-bit non-overlapping copy (compiles to ldr q/str q). let dstp = dst.add(self.d); ptr::copy_nonoverlapping(dstp.sub(offset), dstp, 16); self.d += len; - continue; - } - - if offset >= 8 && len <= 16 { - // For offset 8-15: two 64-bit copies to avoid UB from overlap. + } else if offset >= 8 && len <= 16 { let dstp = dst.add(self.d); let srcp = dstp.sub(offset); ptr::copy_nonoverlapping(srcp, dstp, 8); ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); self.d += len; - continue; - } - - if offset >= 16 { + } else if offset >= 16 { let dstp = dst.add(self.d); wide_copy(dstp.sub(offset), dstp, len); self.d += len; - continue; + } else { + overlapping_copy(dst.add(self.d), offset, len); + self.d += len; } - overlapping_copy(dst.add(self.d), offset, len); - self.d += len; + if !(self.s + 17 <= src_len && self.d + 88 <= dst_len) { + break; + } } Ok(()) } @@ -375,8 +383,6 @@ impl<'s, 'd> Decompress<'s, 'd> { // Find the copy offset and len, then advance the input past the copy. // The rest of this function deals with reading/writing to output only. let entry = TAG_LOOKUP_TABLE.entry(tag_byte); - // Compute num_tag_bytes from tag_type directly to break serial - // dependency through the table entry. let tag_type = (tag_byte & 3) as usize; let num_tag_bytes = tag_type + (tag_type == 3) as usize; let offset = entry.offset_with_ntb(self.src, self.s, num_tag_bytes)?; @@ -503,6 +509,7 @@ impl TagLookupTable { } } + /// Represents a single entry in the tag lookup table. /// /// See the documentation in `TagLookupTable` for the bit layout. @@ -516,6 +523,7 @@ impl TagEntry { self.0 & 0xFF } + /// Return the copy offset corresponding to this copy operation. `s` should /// point to the position just after the tag byte that this entry was read /// from. From c805cfeb8c9f6bc3309830d3fc81e4b15afc4aba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 21:25:05 +0100 Subject: [PATCH 15/41] Snappy tweaks --- src/decompress.rs | 82 ++++++++++++++++++++++------------------------- 1 file changed, 39 insertions(+), 43 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 1b45aaa..e520c54 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -25,6 +25,16 @@ unsafe fn wide_copy(src: *const u8, dst: *mut u8, len: usize) { } } + +/// Extract the offset mask for a given tag_type (1, 2, or 3) using a +/// packed constant, avoiding the dependency chain through num_tag_bytes. +/// Returns a mask: tag_type=1 → 0xFF, tag_type=2 → 0xFFFF, tag_type=3 → 0. +#[inline(always)] +fn extract_offset_mask(tag_type: usize) -> u32 { + const MASKS_PACKED: u64 = 0x0000FFFF00FF0000u64; + ((MASKS_PACKED >> (tag_type * 16)) & 0xFFFF) as u32 +} + /// Copy `len` bytes from `dst - offset` into `dst`, handling overlapping /// regions by expanding with `ptr::copy` until the gap >= 16, then /// switching to non-overlapping 16-byte chunks. @@ -185,6 +195,8 @@ impl<'s, 'd> Decompress<'s, 'd> { Ok(()) } + /// Fast decompression loop using raw pointers for common cases. + /// /// Fast decompression loop using raw pointers for common cases. /// /// The loop condition guarantees sufficient headroom in both source and @@ -200,17 +212,9 @@ impl<'s, 'd> Decompress<'s, 'd> { let src_len = self.src.len(); let dst_len = self.dst.len(); - if !(self.s + 17 <= src_len && self.d + 88 <= dst_len) { - return Ok(()); - } - - // Software-pipelined loop: pre-load the current tag byte and its - // table entry so that after each copy/literal the next iteration's - // loads are already in flight. - let mut byte = *src.add(self.s); - let mut entry_val = TAG_LOOKUP_TABLE.0[byte as usize] as usize; + while self.s + 17 <= src_len && self.d + 88 <= dst_len { + let byte = *src.add(self.s); - loop { if byte & 3 == 0 { let len = (byte >> 2) as usize + 1; if len <= 16 { @@ -222,33 +226,33 @@ impl<'s, 'd> Decompress<'s, 'd> { ); self.s += len; self.d += len; - } else if len <= 60 && self.s + 1 + len + 16 <= src_len { + continue; + } + if len <= 60 && self.s + 1 + len + 16 <= src_len { self.s += 1; wide_copy(src.add(self.s), dst.add(self.d), len); self.s += len; self.d += len; - } else { - self.s += 1; - self.read_literal(len)?; - } - // Re-check loop condition, then pre-load next tag. - if !(self.s + 17 <= src_len && self.d + 88 <= dst_len) { - break; + continue; } - byte = *src.add(self.s); - entry_val = TAG_LOOKUP_TABLE.0[byte as usize] as usize; + self.s += 1; + self.read_literal(len)?; continue; } + let entry_val = TAG_LOOKUP_TABLE.0[byte as usize] as usize; let tag_type = (byte & 3) as usize; let num_tag_bytes = tag_type + (tag_type == 3) as usize; let len = entry_val & 0xFF; self.s += 1; - let mask = u32::MAX >> ((4 - num_tag_bytes as u32) << 3); - let trailer = bytes::loadu_u32_le(src.add(self.s)) as usize - & mask as usize; - let offset = (entry_val & 0x700) | trailer; + // Packed constant for offset mask (C++ ExtractOffset style). + // Replaces the dependency chain tag_type → ntb → shift → mask + // with a shorter tag_type → shift → AND. + let trailer = bytes::loadu_u32_le(src.add(self.s)); + let extracted = + (trailer & extract_offset_mask(tag_type)) as usize; + let offset = (entry_val & 0x700) | extracted; self.s += num_tag_bytes; if self.d <= offset.wrapping_sub(1) { @@ -258,39 +262,31 @@ impl<'s, 'd> Decompress<'s, 'd> { }); } - // Pre-load next tag byte and table entry BEFORE the copy, - // so the ~4-cycle table lookup overlaps with the copy operation. - // Safety: self.s is valid (we'll re-check the loop condition - // after the copy, but reading one byte past the end is safe - // because src_len >= s + 17 held at loop entry and we advanced - // at most 5 bytes). - byte = *src.add(self.s); - entry_val = TAG_LOOKUP_TABLE.0[byte as usize] as usize; - - // All destination bounds are guaranteed by the loop condition - // (d + 88 <= dst_len covers max copy 64 + 24 wiggle room). if len <= 16 && offset >= 16 { let dstp = dst.add(self.d); ptr::copy_nonoverlapping(dstp.sub(offset), dstp, 16); self.d += len; - } else if offset >= 8 && len <= 16 { + continue; + } + + if offset >= 8 && len <= 16 { let dstp = dst.add(self.d); let srcp = dstp.sub(offset); ptr::copy_nonoverlapping(srcp, dstp, 8); ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); self.d += len; - } else if offset >= 16 { + continue; + } + + if offset >= 16 { let dstp = dst.add(self.d); wide_copy(dstp.sub(offset), dstp, len); self.d += len; - } else { - overlapping_copy(dst.add(self.d), offset, len); - self.d += len; + continue; } - if !(self.s + 17 <= src_len && self.d + 88 <= dst_len) { - break; - } + overlapping_copy(dst.add(self.d), offset, len); + self.d += len; } Ok(()) } From 301f9c966f39b9ee06089dce29c9ed21fcbf5b6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 21:30:44 +0100 Subject: [PATCH 16/41] Snappy tweaks --- src/decompress.rs | 70 +++++++++++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index e520c54..8d86e88 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -212,8 +212,17 @@ impl<'s, 'd> Decompress<'s, 'd> { let src_len = self.src.len(); let dst_len = self.dst.len(); - while self.s + 17 <= src_len && self.d + 88 <= dst_len { - let byte = *src.add(self.s); + if !(self.s + 17 <= src_len && self.d + 88 <= dst_len) { + return Ok(()); + } + + // C++ preload trick: carry the current tag byte forward from + // the previous iteration's trailer load, avoiding a separate + // memory load per tag. + let mut preload = *src.add(self.s) as u32; + + loop { + let byte = preload as u8; if byte & 3 == 0 { let len = (byte >> 2) as usize + 1; @@ -226,17 +235,20 @@ impl<'s, 'd> Decompress<'s, 'd> { ); self.s += len; self.d += len; - continue; - } - if len <= 60 && self.s + 1 + len + 16 <= src_len { + } else if len <= 60 && self.s + 1 + len + 16 <= src_len { self.s += 1; wide_copy(src.add(self.s), dst.add(self.d), len); self.s += len; self.d += len; - continue; + } else { + self.s += 1; + self.read_literal(len)?; } - self.s += 1; - self.read_literal(len)?; + // Literals: can't preload, must reload next tag. + if !(self.s + 17 <= src_len && self.d + 88 <= dst_len) { + break; + } + preload = *src.add(self.s) as u32; continue; } @@ -246,12 +258,10 @@ impl<'s, 'd> Decompress<'s, 'd> { let len = entry_val & 0xFF; self.s += 1; - // Packed constant for offset mask (C++ ExtractOffset style). - // Replaces the dependency chain tag_type → ntb → shift → mask - // with a shorter tag_type → shift → AND. - let trailer = bytes::loadu_u32_le(src.add(self.s)); + // Load 4 bytes: trailer data + (for Copy1/Copy2) next tag byte. + let loaded = bytes::loadu_u32_le(src.add(self.s)); let extracted = - (trailer & extract_offset_mask(tag_type)) as usize; + (loaded & extract_offset_mask(tag_type)) as usize; let offset = (entry_val & 0x700) | extracted; self.s += num_tag_bytes; @@ -266,27 +276,39 @@ impl<'s, 'd> Decompress<'s, 'd> { let dstp = dst.add(self.d); ptr::copy_nonoverlapping(dstp.sub(offset), dstp, 16); self.d += len; - continue; - } - - if offset >= 8 && len <= 16 { + } else if offset >= 8 && len <= 16 { let dstp = dst.add(self.d); let srcp = dstp.sub(offset); ptr::copy_nonoverlapping(srcp, dstp, 8); ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); self.d += len; - continue; - } - - if offset >= 16 { + } else if offset >= 16 { let dstp = dst.add(self.d); wide_copy(dstp.sub(offset), dstp, len); self.d += len; - continue; + } else { + overlapping_copy(dst.add(self.d), offset, len); + self.d += len; } - overlapping_copy(dst.add(self.d), offset, len); - self.d += len; + // Preload trick: extract next tag byte from the already-loaded + // u32 trailer. For Copy1 (ntb=1): next tag at loaded byte 1. + // For Copy2 (ntb=2): next tag at loaded byte 2. + // Shift by tag_type*8 bits (works for Copy1/Copy2). + // For Copy4 (ntb=4, ~0% of data): shift gives wrong byte, + // so reload. + preload = loaded >> (tag_type as u32 * 8); + if tag_type == 3 { + // Copy4: next tag byte not in our 4-byte load, reload. + if !(self.s + 17 <= src_len && self.d + 88 <= dst_len) { + break; + } + preload = *src.add(self.s) as u32; + } + + if !(self.s + 17 <= src_len && self.d + 88 <= dst_len) { + break; + } } Ok(()) } From 336041c8dcabacab3404d8bebdbccde6548f3e9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 21:42:41 +0100 Subject: [PATCH 17/41] Snappy tweaks --- src/decompress.rs | 117 +++++++++++++++++++++++++++------------------- 1 file changed, 70 insertions(+), 47 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 8d86e88..00c00bc 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -252,58 +252,81 @@ impl<'s, 'd> Decompress<'s, 'd> { continue; } - let entry_val = TAG_LOOKUP_TABLE.0[byte as usize] as usize; - let tag_type = (byte & 3) as usize; - let num_tag_bytes = tag_type + (tag_type == 3) as usize; - let len = entry_val & 0xFF; - self.s += 1; + if byte & 3 == 2 { + // Copy2: most common copy tag (~55% of URL data). + // Compute len and offset directly, no table lookup needed. + let len = 1 + ((byte >> 2) as usize); + self.s += 1; + let loaded = bytes::loadu_u32_le(src.add(self.s)); + let offset = (loaded & 0xFFFF) as usize; + self.s += 2; + + if self.d <= offset.wrapping_sub(1) { + return Err(Error::Offset { + offset: offset as u64, + dst_pos: self.d as u64, + }); + } - // Load 4 bytes: trailer data + (for Copy1/Copy2) next tag byte. - let loaded = bytes::loadu_u32_le(src.add(self.s)); - let extracted = - (loaded & extract_offset_mask(tag_type)) as usize; - let offset = (entry_val & 0x700) | extracted; - self.s += num_tag_bytes; - - if self.d <= offset.wrapping_sub(1) { - return Err(Error::Offset { - offset: offset as u64, - dst_pos: self.d as u64, - }); - } + if len <= 16 && offset >= 8 { + let dstp = dst.add(self.d); + let srcp = dstp.sub(offset); + ptr::copy_nonoverlapping(srcp, dstp, 8); + ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); + self.d += len; + } else if offset >= 16 { + let dstp = dst.add(self.d); + wide_copy(dstp.sub(offset), dstp, len); + self.d += len; + } else { + overlapping_copy(dst.add(self.d), offset, len); + self.d += len; + } - if len <= 16 && offset >= 16 { - let dstp = dst.add(self.d); - ptr::copy_nonoverlapping(dstp.sub(offset), dstp, 16); - self.d += len; - } else if offset >= 8 && len <= 16 { - let dstp = dst.add(self.d); - let srcp = dstp.sub(offset); - ptr::copy_nonoverlapping(srcp, dstp, 8); - ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); - self.d += len; - } else if offset >= 16 { - let dstp = dst.add(self.d); - wide_copy(dstp.sub(offset), dstp, len); - self.d += len; + preload = loaded >> 16; } else { - overlapping_copy(dst.add(self.d), offset, len); - self.d += len; - } + // Copy1 (~13.5%) and Copy4 (~0%). + let entry_val = TAG_LOOKUP_TABLE.0[byte as usize] as usize; + let tag_type = (byte & 3) as usize; + let num_tag_bytes = tag_type + (tag_type == 3) as usize; + let len = entry_val & 0xFF; + self.s += 1; + + let loaded = bytes::loadu_u32_le(src.add(self.s)); + let extracted = + (loaded & extract_offset_mask(tag_type)) as usize; + let offset = (entry_val & 0x700) | extracted; + self.s += num_tag_bytes; + + if self.d <= offset.wrapping_sub(1) { + return Err(Error::Offset { + offset: offset as u64, + dst_pos: self.d as u64, + }); + } - // Preload trick: extract next tag byte from the already-loaded - // u32 trailer. For Copy1 (ntb=1): next tag at loaded byte 1. - // For Copy2 (ntb=2): next tag at loaded byte 2. - // Shift by tag_type*8 bits (works for Copy1/Copy2). - // For Copy4 (ntb=4, ~0% of data): shift gives wrong byte, - // so reload. - preload = loaded >> (tag_type as u32 * 8); - if tag_type == 3 { - // Copy4: next tag byte not in our 4-byte load, reload. - if !(self.s + 17 <= src_len && self.d + 88 <= dst_len) { - break; + if len <= 16 && offset >= 8 { + let dstp = dst.add(self.d); + let srcp = dstp.sub(offset); + ptr::copy_nonoverlapping(srcp, dstp, 8); + ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); + self.d += len; + } else if offset >= 16 { + let dstp = dst.add(self.d); + wide_copy(dstp.sub(offset), dstp, len); + self.d += len; + } else { + overlapping_copy(dst.add(self.d), offset, len); + self.d += len; + } + + preload = loaded >> (tag_type as u32 * 8); + if tag_type == 3 { + if !(self.s + 17 <= src_len && self.d + 88 <= dst_len) { + break; + } + preload = *src.add(self.s) as u32; } - preload = *src.add(self.s) as u32; } if !(self.s + 17 <= src_len && self.d + 88 <= dst_len) { From 2f890eac7054de278b7998cf0e162921993e277b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 21:53:41 +0100 Subject: [PATCH 18/41] Snappy tweaks --- src/decompress.rs | 118 ++++++++++++++++++---------------------------- 1 file changed, 45 insertions(+), 73 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 00c00bc..387d739 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -212,9 +212,12 @@ impl<'s, 'd> Decompress<'s, 'd> { let src_len = self.src.len(); let dst_len = self.dst.len(); - if !(self.s + 17 <= src_len && self.d + 88 <= dst_len) { + if src_len < 17 || dst_len < 88 { return Ok(()); } + // Precompute loop limits to avoid additions in the hot loop. + let src_limit = src_len - 17; + let dst_limit = dst_len - 88; // C++ preload trick: carry the current tag byte forward from // the previous iteration's trailer load, avoiding a separate @@ -245,91 +248,60 @@ impl<'s, 'd> Decompress<'s, 'd> { self.read_literal(len)?; } // Literals: can't preload, must reload next tag. - if !(self.s + 17 <= src_len && self.d + 88 <= dst_len) { + if !(self.s <= src_limit && self.d <= dst_limit) { break; } preload = *src.add(self.s) as u32; continue; } - if byte & 3 == 2 { - // Copy2: most common copy tag (~55% of URL data). - // Compute len and offset directly, no table lookup needed. - let len = 1 + ((byte >> 2) as usize); - self.s += 1; - let loaded = bytes::loadu_u32_le(src.add(self.s)); - let offset = (loaded & 0xFFFF) as usize; - self.s += 2; - - if self.d <= offset.wrapping_sub(1) { - return Err(Error::Offset { - offset: offset as u64, - dst_pos: self.d as u64, - }); - } + let entry_val = TAG_LOOKUP_TABLE.0[byte as usize] as usize; + let tag_type = (byte & 3) as usize; + let num_tag_bytes = tag_type + (tag_type == 3) as usize; + let len = entry_val & 0xFF; + self.s += 1; - if len <= 16 && offset >= 8 { - let dstp = dst.add(self.d); - let srcp = dstp.sub(offset); - ptr::copy_nonoverlapping(srcp, dstp, 8); - ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); - self.d += len; - } else if offset >= 16 { - let dstp = dst.add(self.d); - wide_copy(dstp.sub(offset), dstp, len); - self.d += len; - } else { - overlapping_copy(dst.add(self.d), offset, len); - self.d += len; - } + // Load 4 bytes: trailer data + (for Copy1/Copy2) next tag byte. + let loaded = bytes::loadu_u32_le(src.add(self.s)); + let extracted = + (loaded & extract_offset_mask(tag_type)) as usize; + let offset = (entry_val & 0x700) | extracted; + self.s += num_tag_bytes; + + if self.d <= offset.wrapping_sub(1) { + return Err(Error::Offset { + offset: offset as u64, + dst_pos: self.d as u64, + }); + } - preload = loaded >> 16; + if len <= 16 && offset >= 8 { + let dstp = dst.add(self.d); + let srcp = dstp.sub(offset); + ptr::copy_nonoverlapping(srcp, dstp, 8); + ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); + self.d += len; + } else if offset >= 16 { + let dstp = dst.add(self.d); + wide_copy(dstp.sub(offset), dstp, len); + self.d += len; } else { - // Copy1 (~13.5%) and Copy4 (~0%). - let entry_val = TAG_LOOKUP_TABLE.0[byte as usize] as usize; - let tag_type = (byte & 3) as usize; - let num_tag_bytes = tag_type + (tag_type == 3) as usize; - let len = entry_val & 0xFF; - self.s += 1; - - let loaded = bytes::loadu_u32_le(src.add(self.s)); - let extracted = - (loaded & extract_offset_mask(tag_type)) as usize; - let offset = (entry_val & 0x700) | extracted; - self.s += num_tag_bytes; - - if self.d <= offset.wrapping_sub(1) { - return Err(Error::Offset { - offset: offset as u64, - dst_pos: self.d as u64, - }); - } - - if len <= 16 && offset >= 8 { - let dstp = dst.add(self.d); - let srcp = dstp.sub(offset); - ptr::copy_nonoverlapping(srcp, dstp, 8); - ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); - self.d += len; - } else if offset >= 16 { - let dstp = dst.add(self.d); - wide_copy(dstp.sub(offset), dstp, len); - self.d += len; - } else { - overlapping_copy(dst.add(self.d), offset, len); - self.d += len; - } + overlapping_copy(dst.add(self.d), offset, len); + self.d += len; + } - preload = loaded >> (tag_type as u32 * 8); - if tag_type == 3 { - if !(self.s + 17 <= src_len && self.d + 88 <= dst_len) { - break; - } - preload = *src.add(self.s) as u32; + // Preload trick: extract next tag byte from the already-loaded + // u32 trailer. For Copy1/Copy2, shift by tag_type*8 bits. + preload = loaded >> (tag_type as u32 * 8); + if tag_type == 3 { + // Copy4: next tag byte not in our 4-byte load, reload. + if !(self.s <= src_limit && self.d <= dst_limit) { + break; } + preload = *src.add(self.s) as u32; } - if !(self.s + 17 <= src_len && self.d + 88 <= dst_len) { + if !(self.s <= src_limit && self.d <= dst_limit) { break; } } From 85eb3b649fee00c645656d828c634b517060d513 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 22:00:11 +0100 Subject: [PATCH 19/41] Snappy tweaks --- src/decompress.rs | 100 ++++++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 47 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 387d739..74a2aea 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -208,50 +208,55 @@ impl<'s, 'd> Decompress<'s, 'd> { #[inline(always)] unsafe fn decompress_fast(&mut self) -> Result<()> { let src = self.src.as_ptr(); - let dst = self.dst.as_mut_ptr(); + let dst_base = self.dst.as_mut_ptr(); let src_len = self.src.len(); let dst_len = self.dst.len(); if src_len < 17 || dst_len < 88 { return Ok(()); } - // Precompute loop limits to avoid additions in the hot loop. - let src_limit = src_len - 17; - let dst_limit = dst_len - 88; - // C++ preload trick: carry the current tag byte forward from - // the previous iteration's trailer load, avoiding a separate - // memory load per tag. - let mut preload = *src.add(self.s) as u32; + // Use raw pointers for the hot loop to avoid base+offset additions. + let mut ip = src.add(self.s); + let mut op = dst_base.add(self.d); + let ip_limit = src.add(src_len - 17); + let op_limit = dst_base.add(dst_len - 88); + + let mut preload = *ip as u32; loop { + // Hint to the compiler that preload is already zero-extended, + // avoiding a redundant AND instruction on aarch64 (LLVM bug 51317). + #[cfg(target_arch = "aarch64")] + core::arch::asm!("", in(reg) preload, options(nomem, nostack, preserves_flags)); + let byte = preload as u8; if byte & 3 == 0 { let len = (byte >> 2) as usize + 1; + ip = ip.add(1); if len <= 16 { - self.s += 1; - ptr::copy_nonoverlapping( - src.add(self.s), - dst.add(self.d), - 16, - ); - self.s += len; - self.d += len; - } else if len <= 60 && self.s + 1 + len + 16 <= src_len { - self.s += 1; - wide_copy(src.add(self.s), dst.add(self.d), len); - self.s += len; - self.d += len; + ptr::copy_nonoverlapping(ip, op, 16); + ip = ip.add(len); + op = op.add(len); + } else if len <= 60 + && (ip as usize + len + 16) <= (src.add(src_len) as usize) + { + wide_copy(ip, op, len); + ip = ip.add(len); + op = op.add(len); } else { - self.s += 1; + // Fall back to index-based slow path. + self.s = ip.offset_from(src) as usize; + self.d = op.offset_from(dst_base) as usize; self.read_literal(len)?; + ip = src.add(self.s); + op = dst_base.add(self.d); } - // Literals: can't preload, must reload next tag. - if !(self.s <= src_limit && self.d <= dst_limit) { + if !(ip <= ip_limit && op <= op_limit) { break; } - preload = *src.add(self.s) as u32; + preload = *ip as u32; continue; } @@ -259,16 +264,20 @@ impl<'s, 'd> Decompress<'s, 'd> { let tag_type = (byte & 3) as usize; let num_tag_bytes = tag_type + (tag_type == 3) as usize; let len = entry_val & 0xFF; - self.s += 1; + ip = ip.add(1); - // Load 4 bytes: trailer data + (for Copy1/Copy2) next tag byte. - let loaded = bytes::loadu_u32_le(src.add(self.s)); + let loaded = bytes::loadu_u32_le(ip); let extracted = (loaded & extract_offset_mask(tag_type)) as usize; let offset = (entry_val & 0x700) | extracted; - self.s += num_tag_bytes; - - if self.d <= offset.wrapping_sub(1) { + ip = ip.add(num_tag_bytes); + + // Check: op - offset >= dst_base (i.e. copy source is valid). + if (op as usize).wrapping_sub(offset) < dst_base as usize + || offset == 0 + { + self.s = ip.offset_from(src) as usize; + self.d = op.offset_from(dst_base) as usize; return Err(Error::Offset { offset: offset as u64, dst_pos: self.d as u64, @@ -276,35 +285,32 @@ impl<'s, 'd> Decompress<'s, 'd> { } if len <= 16 && offset >= 8 { - let dstp = dst.add(self.d); - let srcp = dstp.sub(offset); - ptr::copy_nonoverlapping(srcp, dstp, 8); - ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); - self.d += len; + let srcp = op.sub(offset); + ptr::copy_nonoverlapping(srcp, op, 8); + ptr::copy_nonoverlapping(srcp.add(8), op.add(8), 8); + op = op.add(len); } else if offset >= 16 { - let dstp = dst.add(self.d); - wide_copy(dstp.sub(offset), dstp, len); - self.d += len; + wide_copy(op.sub(offset), op, len); + op = op.add(len); } else { - overlapping_copy(dst.add(self.d), offset, len); - self.d += len; + overlapping_copy(op, offset, len); + op = op.add(len); } - // Preload trick: extract next tag byte from the already-loaded - // u32 trailer. For Copy1/Copy2, shift by tag_type*8 bits. preload = loaded >> (tag_type as u32 * 8); if tag_type == 3 { - // Copy4: next tag byte not in our 4-byte load, reload. - if !(self.s <= src_limit && self.d <= dst_limit) { + if !(ip <= ip_limit && op <= op_limit) { break; } - preload = *src.add(self.s) as u32; + preload = *ip as u32; } - if !(self.s <= src_limit && self.d <= dst_limit) { + if !(ip <= ip_limit && op <= op_limit) { break; } } + self.s = ip.offset_from(src) as usize; + self.d = op.offset_from(dst_base) as usize; Ok(()) } From f20e995f9d603ba8a1722ff15d7bc68e9c858962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 22:01:18 +0100 Subject: [PATCH 20/41] Snappy tweaks --- src/decompress.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 74a2aea..aebf5e6 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -225,11 +225,6 @@ impl<'s, 'd> Decompress<'s, 'd> { let mut preload = *ip as u32; loop { - // Hint to the compiler that preload is already zero-extended, - // avoiding a redundant AND instruction on aarch64 (LLVM bug 51317). - #[cfg(target_arch = "aarch64")] - core::arch::asm!("", in(reg) preload, options(nomem, nostack, preserves_flags)); - let byte = preload as u8; if byte & 3 == 0 { From f8771bc3b6db0097ee897c6e1f7dcf630699530d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 22:15:19 +0100 Subject: [PATCH 21/41] Snappy tweaks --- src/decompress.rs | 118 ++++++++++++++++++++++++---------------------- 1 file changed, 61 insertions(+), 57 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index aebf5e6..047cb3c 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -9,6 +9,7 @@ use crate::MAX_INPUT_SIZE; /// tag byte. const TAG_LOOKUP_TABLE: TagLookupTable = TagLookupTable(tag::TAG_LOOKUP_TABLE); + /// Copy up to 64 bytes using unrolled 16-byte copies. /// `src` and `dst` must not overlap and must have at least `len + 16` bytes /// of addressable memory (we always copy in 16-byte chunks). @@ -227,82 +228,85 @@ impl<'s, 'd> Decompress<'s, 'd> { loop { let byte = preload as u8; - if byte & 3 == 0 { - let len = (byte >> 2) as usize + 1; + // Copy path first — uses `continue` to branch back. + // Literal path last — falls through to loop back-edge, + // saving one unconditional branch (PGO-informed layout). + if byte & 3 != 0 { + let entry_val = TAG_LOOKUP_TABLE.0[byte as usize] as usize; + let tag_type = (byte & 3) as usize; + let num_tag_bytes = tag_type + (tag_type == 3) as usize; + let len = entry_val & 0xFF; ip = ip.add(1); - if len <= 16 { - ptr::copy_nonoverlapping(ip, op, 16); - ip = ip.add(len); - op = op.add(len); - } else if len <= 60 - && (ip as usize + len + 16) <= (src.add(src_len) as usize) + + let loaded = bytes::loadu_u32_le(ip); + let extracted = + (loaded & extract_offset_mask(tag_type)) as usize; + let offset = (entry_val & 0x700) | extracted; + ip = ip.add(num_tag_bytes); + + if (op as usize).wrapping_sub(offset) < dst_base as usize + || offset == 0 { - wide_copy(ip, op, len); - ip = ip.add(len); - op = op.add(len); - } else { - // Fall back to index-based slow path. self.s = ip.offset_from(src) as usize; self.d = op.offset_from(dst_base) as usize; - self.read_literal(len)?; - ip = src.add(self.s); - op = dst_base.add(self.d); + return Err(Error::Offset { + offset: offset as u64, + dst_pos: self.d as u64, + }); + } + + if len <= 16 && offset >= 8 { + let srcp = op.sub(offset); + ptr::copy_nonoverlapping(srcp, op, 8); + ptr::copy_nonoverlapping(srcp.add(8), op.add(8), 8); + op = op.add(len); + } else if offset >= 16 { + wide_copy(op.sub(offset), op, len); + op = op.add(len); + } else { + overlapping_copy(op, offset, len); + op = op.add(len); } + + preload = loaded >> (tag_type as u32 * 8); + if tag_type == 3 { + if !(ip <= ip_limit && op <= op_limit) { + break; + } + preload = *ip as u32; + } + if !(ip <= ip_limit && op <= op_limit) { break; } - preload = *ip as u32; continue; } - let entry_val = TAG_LOOKUP_TABLE.0[byte as usize] as usize; - let tag_type = (byte & 3) as usize; - let num_tag_bytes = tag_type + (tag_type == 3) as usize; - let len = entry_val & 0xFF; + // Literal path — at end of loop body so preload falls + // through to loop back-edge without an extra branch. + let len = (byte >> 2) as usize + 1; ip = ip.add(1); - - let loaded = bytes::loadu_u32_le(ip); - let extracted = - (loaded & extract_offset_mask(tag_type)) as usize; - let offset = (entry_val & 0x700) | extracted; - ip = ip.add(num_tag_bytes); - - // Check: op - offset >= dst_base (i.e. copy source is valid). - if (op as usize).wrapping_sub(offset) < dst_base as usize - || offset == 0 - { - self.s = ip.offset_from(src) as usize; - self.d = op.offset_from(dst_base) as usize; - return Err(Error::Offset { - offset: offset as u64, - dst_pos: self.d as u64, - }); - } - - if len <= 16 && offset >= 8 { - let srcp = op.sub(offset); - ptr::copy_nonoverlapping(srcp, op, 8); - ptr::copy_nonoverlapping(srcp.add(8), op.add(8), 8); + if len <= 16 { + ptr::copy_nonoverlapping(ip, op, 16); + ip = ip.add(len); op = op.add(len); - } else if offset >= 16 { - wide_copy(op.sub(offset), op, len); + } else if len <= 60 + && (ip as usize + len + 16) <= (src.add(src_len) as usize) + { + wide_copy(ip, op, len); + ip = ip.add(len); op = op.add(len); } else { - overlapping_copy(op, offset, len); - op = op.add(len); - } - - preload = loaded >> (tag_type as u32 * 8); - if tag_type == 3 { - if !(ip <= ip_limit && op <= op_limit) { - break; - } - preload = *ip as u32; + self.s = ip.offset_from(src) as usize; + self.d = op.offset_from(dst_base) as usize; + self.read_literal(len)?; + ip = src.add(self.s); + op = dst_base.add(self.d); } - if !(ip <= ip_limit && op <= op_limit) { break; } + preload = *ip as u32; } self.s = ip.offset_from(src) as usize; self.d = op.offset_from(dst_base) as usize; From 22732a963a6ad6badef5be0a801506402d5ca1cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 22:21:57 +0100 Subject: [PATCH 22/41] Snappy tweaks --- src/decompress.rs | 64 +++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 35 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 047cb3c..9d2e75f 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -227,10 +227,10 @@ impl<'s, 'd> Decompress<'s, 'd> { loop { let byte = preload as u8; + // Track whether we need to reload preload from memory + // (literals always, Copy4 always, Copy1/Copy2 never). + let mut reload = true; - // Copy path first — uses `continue` to branch back. - // Literal path last — falls through to loop back-edge, - // saving one unconditional branch (PGO-informed layout). if byte & 3 != 0 { let entry_val = TAG_LOOKUP_TABLE.0[byte as usize] as usize; let tag_type = (byte & 3) as usize; @@ -268,45 +268,39 @@ impl<'s, 'd> Decompress<'s, 'd> { op = op.add(len); } + // Preload next tag from the trailer for Copy1/Copy2. preload = loaded >> (tag_type as u32 * 8); - if tag_type == 3 { - if !(ip <= ip_limit && op <= op_limit) { - break; - } - preload = *ip as u32; - } - - if !(ip <= ip_limit && op <= op_limit) { - break; + reload = tag_type == 3; + } else { + let len = (byte >> 2) as usize + 1; + ip = ip.add(1); + if len <= 16 { + ptr::copy_nonoverlapping(ip, op, 16); + ip = ip.add(len); + op = op.add(len); + } else if len <= 60 + && (ip as usize + len + 16) + <= (src.add(src_len) as usize) + { + wide_copy(ip, op, len); + ip = ip.add(len); + op = op.add(len); + } else { + self.s = ip.offset_from(src) as usize; + self.d = op.offset_from(dst_base) as usize; + self.read_literal(len)?; + ip = src.add(self.s); + op = dst_base.add(self.d); } - continue; } - // Literal path — at end of loop body so preload falls - // through to loop back-edge without an extra branch. - let len = (byte >> 2) as usize + 1; - ip = ip.add(1); - if len <= 16 { - ptr::copy_nonoverlapping(ip, op, 16); - ip = ip.add(len); - op = op.add(len); - } else if len <= 60 - && (ip as usize + len + 16) <= (src.add(src_len) as usize) - { - wide_copy(ip, op, len); - ip = ip.add(len); - op = op.add(len); - } else { - self.s = ip.offset_from(src) as usize; - self.d = op.offset_from(dst_base) as usize; - self.read_literal(len)?; - ip = src.add(self.s); - op = dst_base.add(self.d); - } + // Single unified bounds check and preload for all paths. if !(ip <= ip_limit && op <= op_limit) { break; } - preload = *ip as u32; + if reload { + preload = *ip as u32; + } } self.s = ip.offset_from(src) as usize; self.d = op.offset_from(dst_base) as usize; From fe4a9714cb4a69bf3674c878b8021e3732f52f2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 11 Mar 2026 22:31:00 +0100 Subject: [PATCH 23/41] Snappy tweaks --- src/decompress.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 9d2e75f..896f1e9 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -222,6 +222,8 @@ impl<'s, 'd> Decompress<'s, 'd> { let mut op = dst_base.add(self.d); let ip_limit = src.add(src_len - 17); let op_limit = dst_base.add(dst_len - 88); + let src_end = src.add(src_len); + let dst_base_addr = dst_base as usize; let mut preload = *ip as u32; @@ -244,7 +246,7 @@ impl<'s, 'd> Decompress<'s, 'd> { let offset = (entry_val & 0x700) | extracted; ip = ip.add(num_tag_bytes); - if (op as usize).wrapping_sub(offset) < dst_base as usize + if (op as usize).wrapping_sub(offset) < dst_base_addr || offset == 0 { self.s = ip.offset_from(src) as usize; @@ -280,7 +282,7 @@ impl<'s, 'd> Decompress<'s, 'd> { op = op.add(len); } else if len <= 60 && (ip as usize + len + 16) - <= (src.add(src_len) as usize) + <= (src_end as usize) { wide_copy(ip, op, len); ip = ip.add(len); From 15b17f99fcc3745332856c11ed58cffc5471757c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 12 Mar 2026 06:15:23 +0100 Subject: [PATCH 24/41] Snappy tweaks --- src/decompress.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decompress.rs b/src/decompress.rs index 896f1e9..1950afe 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -297,7 +297,7 @@ impl<'s, 'd> Decompress<'s, 'd> { } // Single unified bounds check and preload for all paths. - if !(ip <= ip_limit && op <= op_limit) { + if ip > ip_limit || op > op_limit { break; } if reload { From 7409b58fb66328a8887db65cb5dd9bdb2d27b575 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 12 Mar 2026 07:25:58 +0100 Subject: [PATCH 25/41] Snappy tweaks --- src/decompress.rs | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 1950afe..9d2ead2 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -11,8 +11,8 @@ const TAG_LOOKUP_TABLE: TagLookupTable = TagLookupTable(tag::TAG_LOOKUP_TABLE); /// Copy up to 64 bytes using unrolled 16-byte copies. -/// `src` and `dst` must not overlap and must have at least `len + 16` bytes -/// of addressable memory (we always copy in 16-byte chunks). +/// `src` and `dst` must not overlap in each 16-byte chunk. +/// Use `wide_copy_long` when src and dst are guaranteed >= 32 apart. #[inline(always)] unsafe fn wide_copy(src: *const u8, dst: *mut u8, len: usize) { debug_assert!(len <= 64); @@ -26,6 +26,17 @@ unsafe fn wide_copy(src: *const u8, dst: *mut u8, len: usize) { } } +/// Copy up to 64 bytes using 32-byte copies (ldp/stp q pairs on ARM). +/// Requires src and dst to be at least 32 bytes apart (no overlap). +#[inline(always)] +unsafe fn wide_copy_long(src: *const u8, dst: *mut u8, len: usize) { + debug_assert!(len <= 64); + ptr::copy_nonoverlapping(src, dst, 32); + if len > 32 { + ptr::copy_nonoverlapping(src.add(32), dst.add(32), 32); + } +} + /// Extract the offset mask for a given tag_type (1, 2, or 3) using a /// packed constant, avoiding the dependency chain through num_tag_bytes. @@ -262,6 +273,9 @@ impl<'s, 'd> Decompress<'s, 'd> { ptr::copy_nonoverlapping(srcp, op, 8); ptr::copy_nonoverlapping(srcp.add(8), op.add(8), 8); op = op.add(len); + } else if offset >= 32 { + wide_copy_long(op.sub(offset), op, len); + op = op.add(len); } else if offset >= 16 { wide_copy(op.sub(offset), op, len); op = op.add(len); @@ -284,7 +298,7 @@ impl<'s, 'd> Decompress<'s, 'd> { && (ip as usize + len + 16) <= (src_end as usize) { - wide_copy(ip, op, len); + wide_copy_long(ip, op, len); ip = ip.add(len); op = op.add(len); } else { From 24438ee172adedc0f50c979b6f8b2b1e1a5fdf52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 12 Mar 2026 07:58:21 +0100 Subject: [PATCH 26/41] Snappy tweaks x86 --- src/decompress.rs | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 9d2ead2..0d2ccdb 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -38,13 +38,23 @@ unsafe fn wide_copy_long(src: *const u8, dst: *mut u8, len: usize) { } -/// Extract the offset mask for a given tag_type (1, 2, or 3) using a -/// packed constant, avoiding the dependency chain through num_tag_bytes. +/// Extract the offset mask for a given tag_type (1, 2, or 3). /// Returns a mask: tag_type=1 → 0xFF, tag_type=2 → 0xFFFF, tag_type=3 → 0. +/// +/// On ARM, uses a packed u64 constant with shift (avoids memory load). +/// On x86, uses an array lookup (avoids 10-byte movabs + variable shift). #[inline(always)] fn extract_offset_mask(tag_type: usize) -> u32 { - const MASKS_PACKED: u64 = 0x0000FFFF00FF0000u64; - ((MASKS_PACKED >> (tag_type * 16)) & 0xFFFF) as u32 + #[cfg(target_arch = "aarch64")] + { + const MASKS_PACKED: u64 = 0x0000FFFF00FF0000u64; + ((MASKS_PACKED >> (tag_type * 16)) & 0xFFFF) as u32 + } + #[cfg(not(target_arch = "aarch64"))] + { + const MASKS: [u32; 4] = [0, 0xFF, 0xFFFF, 0]; + MASKS[tag_type] + } } /// Copy `len` bytes from `dst - offset` into `dst`, handling overlapping @@ -257,9 +267,9 @@ impl<'s, 'd> Decompress<'s, 'd> { let offset = (entry_val & 0x700) | extracted; ip = ip.add(num_tag_bytes); - if (op as usize).wrapping_sub(offset) < dst_base_addr - || offset == 0 - { + // Compute copy source once; reuse for bounds check and copy. + let srcp = op.sub(offset); + if (srcp as usize) < dst_base_addr || offset == 0 { self.s = ip.offset_from(src) as usize; self.d = op.offset_from(dst_base) as usize; return Err(Error::Offset { @@ -269,15 +279,14 @@ impl<'s, 'd> Decompress<'s, 'd> { } if len <= 16 && offset >= 8 { - let srcp = op.sub(offset); ptr::copy_nonoverlapping(srcp, op, 8); ptr::copy_nonoverlapping(srcp.add(8), op.add(8), 8); op = op.add(len); } else if offset >= 32 { - wide_copy_long(op.sub(offset), op, len); + wide_copy_long(srcp, op, len); op = op.add(len); } else if offset >= 16 { - wide_copy(op.sub(offset), op, len); + wide_copy(srcp, op, len); op = op.add(len); } else { overlapping_copy(op, offset, len); From aacb7c512c0c5a932be76e385e8a5418e9035601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 12 Mar 2026 09:37:50 +0100 Subject: [PATCH 27/41] Snappy tweaks --- src/decompress.rs | 218 +++++++++++++++------------------------------- 1 file changed, 70 insertions(+), 148 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 0d2ccdb..a0c5519 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -6,9 +6,8 @@ use crate::tag; use crate::MAX_INPUT_SIZE; /// A lookup table for quickly computing the various attributes derived from a -/// tag byte. -const TAG_LOOKUP_TABLE: TagLookupTable = TagLookupTable(tag::TAG_LOOKUP_TABLE); - +/// tag byte. See the comment above `TagLookupTable` for the bit layout. +const TAG_LOOKUP_TABLE: [u16; 256] = tag::TAG_LOOKUP_TABLE; /// Copy up to 64 bytes using unrolled 16-byte copies. /// `src` and `dst` must not overlap in each 16-byte chunk. @@ -37,7 +36,6 @@ unsafe fn wide_copy_long(src: *const u8, dst: *mut u8, len: usize) { } } - /// Extract the offset mask for a given tag_type (1, 2, or 3). /// Returns a mask: tag_type=1 → 0xFF, tag_type=2 → 0xFFFF, tag_type=3 → 0. /// @@ -57,6 +55,28 @@ fn extract_offset_mask(tag_type: usize) -> u32 { } } +/// Dispatch a copy of `len` bytes from `dst - offset` to `dst`. +/// +/// Tries fast wide-copy paths (non-overlapping 8/16/32 byte chunks). +/// Falls back to `overlapping_copy` when offset < 16. +/// +/// Caller must ensure at least `len + 24` bytes of writable space at `dst` +/// and at least `offset` valid bytes preceding `dst`. +#[inline(always)] +unsafe fn copy_dispatch(dst: *mut u8, offset: usize, len: usize) { + let srcp = dst.sub(offset); + if len <= 16 && offset >= 8 { + ptr::copy_nonoverlapping(srcp, dst, 8); + ptr::copy_nonoverlapping(srcp.add(8), dst.add(8), 8); + } else if offset >= 32 { + wide_copy_long(srcp, dst, len); + } else if offset >= 16 { + wide_copy(srcp, dst, len); + } else { + overlapping_copy(dst, offset, len); + } +} + /// Copy `len` bytes from `dst - offset` into `dst`, handling overlapping /// regions by expanding with `ptr::copy` until the gap >= 16, then /// switching to non-overlapping 16-byte chunks. @@ -255,7 +275,7 @@ impl<'s, 'd> Decompress<'s, 'd> { let mut reload = true; if byte & 3 != 0 { - let entry_val = TAG_LOOKUP_TABLE.0[byte as usize] as usize; + let entry_val = TAG_LOOKUP_TABLE[byte as usize] as usize; let tag_type = (byte & 3) as usize; let num_tag_bytes = tag_type + (tag_type == 3) as usize; let len = entry_val & 0xFF; @@ -278,22 +298,12 @@ impl<'s, 'd> Decompress<'s, 'd> { }); } - if len <= 16 && offset >= 8 { - ptr::copy_nonoverlapping(srcp, op, 8); - ptr::copy_nonoverlapping(srcp.add(8), op.add(8), 8); - op = op.add(len); - } else if offset >= 32 { - wide_copy_long(srcp, op, len); - op = op.add(len); - } else if offset >= 16 { - wide_copy(srcp, op, len); - op = op.add(len); - } else { - overlapping_copy(op, offset, len); - op = op.add(len); - } + copy_dispatch(op, offset, len); + op = op.add(len); // Preload next tag from the trailer for Copy1/Copy2. + // For Copy4 (num_tag_bytes=4), shift is 32 → result is 0, + // but reload=true overwrites it anyway. preload = loaded >> (tag_type as u32 * 8); reload = tag_type == 3; } else { @@ -304,8 +314,7 @@ impl<'s, 'd> Decompress<'s, 'd> { ip = ip.add(len); op = op.add(len); } else if len <= 60 - && (ip as usize + len + 16) - <= (src_end as usize) + && (ip as usize + len + 16) <= (src_end as usize) { wide_copy_long(ip, op, len); ip = ip.add(len); @@ -417,13 +426,41 @@ impl<'s, 'd> Decompress<'s, 'd> { /// should point to the byte immediately proceding the copy tag byte. #[inline(always)] fn read_copy(&mut self, tag_byte: u8) -> Result<()> { - // Find the copy offset and len, then advance the input past the copy. - // The rest of this function deals with reading/writing to output only. - let entry = TAG_LOOKUP_TABLE.entry(tag_byte); + let entry_val = TAG_LOOKUP_TABLE[tag_byte as usize] as usize; let tag_type = (tag_byte & 3) as usize; let num_tag_bytes = tag_type + (tag_type == 3) as usize; - let offset = entry.offset_with_ntb(self.src, self.s, num_tag_bytes)?; - let len = entry.len(); + let len = entry_val & 0xFF; + + // Read offset from compressed input. + let trailer = if self.s + 4 <= self.src.len() { + unsafe { + let p = self.src.as_ptr().add(self.s); + let mask = u32::MAX >> ((4 - num_tag_bytes as u32) << 3); + bytes::loadu_u32_le(p) as usize & mask as usize + } + } else if num_tag_bytes == 1 { + if self.s >= self.src.len() { + return Err(Error::CopyRead { + len: 1, + src_len: (self.src.len() - self.s) as u64, + }); + } + self.src[self.s] as usize + } else if num_tag_bytes == 2 { + if self.s + 1 >= self.src.len() { + return Err(Error::CopyRead { + len: 2, + src_len: (self.src.len() - self.s) as u64, + }); + } + bytes::read_u16_le(&self.src[self.s..]) as usize + } else { + return Err(Error::CopyRead { + len: num_tag_bytes as u64, + src_len: (self.src.len() - self.s) as u64, + }); + }; + let offset = (entry_val & 0x700) | trailer; self.s += num_tag_bytes; // What we really care about here is whether `d == 0` or `d < offset`. @@ -436,34 +473,14 @@ impl<'s, 'd> Decompress<'s, 'd> { dst_pos: self.d as u64, }); } - // When all is said and done, dst is advanced to end. let end = self.d + len; - // When the copy is small and the offset is at least 8 bytes away from - // `d`, then we can decompress the copy with two 64 bit unaligned - // loads/stores. - if offset >= 8 && len <= 16 && self.d + 16 <= self.dst.len() { + if end + 24 <= self.dst.len() { unsafe { - let dstp = self.dst.as_mut_ptr().add(self.d); - let srcp = dstp.sub(offset); - ptr::copy_nonoverlapping(srcp, dstp, 8); - ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); - } - } else if offset >= 16 && end + 16 <= self.dst.len() { - unsafe { - let dstp = self.dst.as_mut_ptr().add(self.d); - wide_copy(dstp.sub(offset), dstp, len); - } - // If we have some wiggle room, try to decompress the copy 16 bytes - // at a time with 128 bit unaligned loads/stores. Remember, we can't - // just do a memcpy because decompressing copies may require copying - // overlapping memory. - // - // We need the extra wiggle room to make effective use of 128 bit - // loads/stores. Even if the store ends up copying more data than we - // need, we're careful to advance `d` by the correct amount at the end. - } else if end + 24 <= self.dst.len() { - unsafe { - overlapping_copy(self.dst.as_mut_ptr().add(self.d), offset, len); + copy_dispatch( + self.dst.as_mut_ptr().add(self.d), + offset, + len, + ); } } else { if end > self.dst.len() { @@ -472,8 +489,7 @@ impl<'s, 'd> Decompress<'s, 'd> { dst_len: (self.dst.len() - self.d) as u64, }); } - // Finally, the slow byte-by-byte case, which should only be used - // for the last few bytes of decompression. + // Byte-by-byte fallback for the last few bytes. while self.d != end { self.dst[self.d] = self.dst[self.d - offset]; self.d += 1; @@ -515,98 +531,4 @@ impl Header { } } -/// A lookup table for quickly computing the various attributes derived from -/// a tag byte. The attributes are most useful for the three "copy" tags -/// and include the length of the copy, part of the offset (for copy 1-byte -/// only) and the total number of bytes proceding the tag byte that encode -/// the other part of the offset (1 for copy 1, 2 for copy 2 and 4 for copy 4). -/// -/// More specifically, the keys of the table are u8s and the values are u16s. -/// The bits of the values are laid out as follows: -/// -/// xxaa abbb xxcc cccc -/// -/// Where `a` is the number of bytes, `b` are the three bits of the offset -/// for copy 1 (the other 8 bits are in the byte proceding the tag byte; for -/// copy 2 and copy 4, `b = 0`), and `c` is the length of the copy (max of 64). -/// -/// We could pack this in fewer bits, but the position of the three `b` bits -/// lines up with the most significant three bits in the total offset for copy -/// 1, which avoids an extra shift instruction. -/// -/// In sum, this table is useful because it reduces branches and various -/// arithmetic operations. -struct TagLookupTable([u16; 256]); - -impl TagLookupTable { - /// Look up the tag entry given the tag `byte`. - #[inline(always)] - fn entry(&self, byte: u8) -> TagEntry { - TagEntry(self.0[byte as usize] as usize) - } -} - - -/// Represents a single entry in the tag lookup table. -/// -/// See the documentation in `TagLookupTable` for the bit layout. -/// -/// The type is a `usize` for convenience. -struct TagEntry(usize); -impl TagEntry { - /// Return the total copy length, capped at 255. - fn len(&self) -> usize { - self.0 & 0xFF - } - - - /// Return the copy offset corresponding to this copy operation. `s` should - /// point to the position just after the tag byte that this entry was read - /// from. - /// - /// This requires reading from the compressed input since the offset is - /// encoded in bytes proceding the tag byte. - fn offset_with_ntb( - &self, - src: &[u8], - s: usize, - num_tag_bytes: usize, - ) -> Result { - let trailer = - // It is critical for this case to come first, since it is the - // fast path. We really hope that this case gets branch - // predicted. - if s + 4 <= src.len() { - unsafe { - // SAFETY: The conditional above guarantees that - // src[s..s+4] is valid to read from. - let p = src.as_ptr().add(s); - let mask = u32::MAX >> ((4 - num_tag_bytes as u32) << 3); - bytes::loadu_u32_le(p) as usize & mask as usize - } - } else if num_tag_bytes == 1 { - if s >= src.len() { - return Err(Error::CopyRead { - len: 1, - src_len: (src.len() - s) as u64, - }); - } - src[s] as usize - } else if num_tag_bytes == 2 { - if s + 1 >= src.len() { - return Err(Error::CopyRead { - len: 2, - src_len: (src.len() - s) as u64, - }); - } - bytes::read_u16_le(&src[s..]) as usize - } else { - return Err(Error::CopyRead { - len: num_tag_bytes as u64, - src_len: (src.len() - s) as u64, - }); - }; - Ok((self.0 & 0b0000_0111_0000_0000) | trailer) - } -} From 9182f8b50133dbf95ecdcbb778b9612874e64d15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 12 Mar 2026 09:38:59 +0100 Subject: [PATCH 28/41] Snappy tweaks --- src/decompress.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index a0c5519..4cac892 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -476,11 +476,7 @@ impl<'s, 'd> Decompress<'s, 'd> { let end = self.d + len; if end + 24 <= self.dst.len() { unsafe { - copy_dispatch( - self.dst.as_mut_ptr().add(self.d), - offset, - len, - ); + copy_dispatch(self.dst.as_mut_ptr().add(self.d), offset, len); } } else { if end > self.dst.len() { @@ -530,5 +526,3 @@ impl Header { Ok(Header { len: header_len, decompress_len: decompress_len as usize }) } } - - From 09c9c80d1c6c7d52134039c466e75e7cbda45bb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 12 Mar 2026 09:50:58 +0100 Subject: [PATCH 29/41] Minimize --- src/decompress.rs | 54 ++++++----------------------------------------- 1 file changed, 7 insertions(+), 47 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 4cac892..4030e90 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -5,8 +5,10 @@ use crate::error::{Error, Result}; use crate::tag; use crate::MAX_INPUT_SIZE; -/// A lookup table for quickly computing the various attributes derived from a -/// tag byte. See the comment above `TagLookupTable` for the bit layout. +/// Tag lookup table. Keys are tag bytes, values are u16 with bit layout: +/// xxaa abbb xxcc cccc +/// Where `a` = num_tag_bytes (1/2/4), `b` = offset high bits (copy 1 only), +/// `c` = copy length (max 64). const TAG_LOOKUP_TABLE: [u16; 256] = tag::TAG_LOOKUP_TABLE; /// Copy up to 64 bytes using unrolled 16-byte copies. @@ -237,16 +239,8 @@ impl<'s, 'd> Decompress<'s, 'd> { Ok(()) } - /// Fast decompression loop using raw pointers for common cases. - /// - /// Fast decompression loop using raw pointers for common cases. - /// - /// The loop condition guarantees sufficient headroom in both source and - /// destination buffers to eliminate most bounds checks from the loop body: - /// - `s + 17 <= src_len`: ensures 16 bytes of literal data + 1 tag byte - /// can always be read, and 4 bytes of copy offset data (since 17 > 5). - /// - `d + 88 <= dst_len`: ensures max copy (64 bytes) + overlapping_copy - /// wiggle room (24 bytes) always fits, so no destination checks needed. + /// Fast inner loop using raw pointers. Requires 17 bytes of source + /// headroom and 88 bytes of destination headroom to avoid bounds checks. #[inline(always)] unsafe fn decompress_fast(&mut self) -> Result<()> { let src = self.src.as_ptr(); @@ -258,7 +252,6 @@ impl<'s, 'd> Decompress<'s, 'd> { return Ok(()); } - // Use raw pointers for the hot loop to avoid base+offset additions. let mut ip = src.add(self.s); let mut op = dst_base.add(self.d); let ip_limit = src.add(src_len - 17); @@ -270,8 +263,6 @@ impl<'s, 'd> Decompress<'s, 'd> { loop { let byte = preload as u8; - // Track whether we need to reload preload from memory - // (literals always, Copy4 always, Copy1/Copy2 never). let mut reload = true; if byte & 3 != 0 { @@ -287,7 +278,6 @@ impl<'s, 'd> Decompress<'s, 'd> { let offset = (entry_val & 0x700) | extracted; ip = ip.add(num_tag_bytes); - // Compute copy source once; reuse for bounds check and copy. let srcp = op.sub(offset); if (srcp as usize) < dst_base_addr || offset == 0 { self.s = ip.offset_from(src) as usize; @@ -301,9 +291,6 @@ impl<'s, 'd> Decompress<'s, 'd> { copy_dispatch(op, offset, len); op = op.add(len); - // Preload next tag from the trailer for Copy1/Copy2. - // For Copy4 (num_tag_bytes=4), shift is 32 → result is 0, - // but reload=true overwrites it anyway. preload = loaded >> (tag_type as u32 * 8); reload = tag_type == 3; } else { @@ -328,7 +315,6 @@ impl<'s, 'd> Decompress<'s, 'd> { } } - // Single unified bounds check and preload for all paths. if ip > ip_limit || op > op_limit { break; } @@ -355,34 +341,20 @@ impl<'s, 'd> Decompress<'s, 'd> { fn read_literal(&mut self, len: usize) -> Result<()> { debug_assert!(len <= 64); let mut len = len as u64; - // As an optimization for the common case, if the literal length is - // <=16 and we have enough room in both `src` and `dst`, copy the - // literal using unaligned loads and stores. - // - // We pick 16 bytes with the hope that it optimizes down to a 128 bit - // load/store. if len <= 16 && self.s + 16 <= self.src.len() && self.d + 16 <= self.dst.len() { unsafe { - // SAFETY: We know both src and dst have at least 16 bytes of - // wiggle room after s/d, even if `len` is <16, so the copy is - // safe. let srcp = self.src.as_ptr().add(self.s); let dstp = self.dst.as_mut_ptr().add(self.d); - // Hopefully uses SIMD registers for 128 bit load/store. ptr::copy_nonoverlapping(srcp, dstp, 16); } self.d += len as usize; self.s += len as usize; return Ok(()); } - // When the length is bigger than 60, it indicates that we need to read - // an additional 1-4 bytes to get the real length of the literal. if len >= 61 { - // If there aren't at least 4 bytes left to read then we know this - // is corrupt because the literal must have length >=61. if self.s as u64 + 4 > self.src.len() as u64 { return Err(Error::Literal { len: 4, @@ -390,17 +362,12 @@ impl<'s, 'd> Decompress<'s, 'd> { dst_len: (self.dst.len() - self.d) as u64, }); } - // Since we know there are 4 bytes left to read, read a 32 bit LE - // integer and mask away the bits we don't need. let byte_count = len as usize - 60; let mask = u32::MAX >> ((4 - byte_count as u32) << 3); len = bytes::read_u32_le(&self.src[self.s..]) as u64; len = (len & mask as u64) + 1; self.s += byte_count; } - // If there's not enough buffer left to load or store this literal, - // then the input is corrupt. - // if self.s + len > self.src.len() || self.d + len > self.dst.len() { if ((self.src.len() - self.s) as u64) < len || ((self.dst.len() - self.d) as u64) < len { @@ -411,8 +378,6 @@ impl<'s, 'd> Decompress<'s, 'd> { }); } unsafe { - // SAFETY: We've already checked the bounds, so we know this copy - // is correct. let srcp = self.src.as_ptr().add(self.s); let dstp = self.dst.as_mut_ptr().add(self.d); ptr::copy_nonoverlapping(srcp, dstp, len as usize); @@ -422,8 +387,7 @@ impl<'s, 'd> Decompress<'s, 'd> { Ok(()) } - /// Reads a copy from `src` and writes the decompressed bytes to `dst`. `s` - /// should point to the byte immediately proceding the copy tag byte. + /// Reads a copy tag and writes the decompressed bytes. #[inline(always)] fn read_copy(&mut self, tag_byte: u8) -> Result<()> { let entry_val = TAG_LOOKUP_TABLE[tag_byte as usize] as usize; @@ -463,10 +427,6 @@ impl<'s, 'd> Decompress<'s, 'd> { let offset = (entry_val & 0x700) | trailer; self.s += num_tag_bytes; - // What we really care about here is whether `d == 0` or `d < offset`. - // To save an extra branch, use `d < offset - 1` instead. If `d` is - // `0`, then `offset.wrapping_sub(1)` will be usize::MAX which is also - // the max value of `d`. if self.d <= offset.wrapping_sub(1) { return Err(Error::Offset { offset: offset as u64, From a33e7a1f9eda7fa660c8a58208a0109b4cfd91b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 12 Mar 2026 10:05:50 +0100 Subject: [PATCH 30/41] Minimize changes --- src/decompress.rs | 158 ++++++++++++++++++++++++---------------------- 1 file changed, 82 insertions(+), 76 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 4030e90..2d50c15 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -220,15 +220,89 @@ impl<'s, 'd> Decompress<'s, 'd> { unsafe { self.decompress_fast()?; } - while self.s < self.src.len() { - let byte = self.src[self.s]; - self.s += 1; - if byte & 0b000000_11 == 0 { - let len = (byte >> 2) as usize + 1; - self.read_literal(len)?; - } else { - self.read_copy(byte)?; + // Slow tail: process remaining bytes with bounds checks. + // Uses raw pointers like the fast path for consistency. + unsafe { + let src = self.src.as_ptr(); + let dst_base = self.dst.as_mut_ptr(); + let src_end = src.add(self.src.len()); + let dst_end = dst_base.add(self.dst.len()); + let dst_base_addr = dst_base as usize; + let mut ip = src.add(self.s); + let mut op = dst_base.add(self.d); + + while ip < src_end { + let byte = *ip; + ip = ip.add(1); + + if byte & 3 == 0 { + let len = (byte >> 2) as usize + 1; + self.s = ip.offset_from(src) as usize; + self.d = op.offset_from(dst_base) as usize; + self.read_literal(len)?; + ip = src.add(self.s); + op = dst_base.add(self.d); + } else { + let entry_val = TAG_LOOKUP_TABLE[byte as usize] as usize; + let tag_type = (byte & 3) as usize; + let num_tag_bytes = tag_type + (tag_type == 3) as usize; + let len = entry_val & 0xFF; + + if ip.add(num_tag_bytes) > src_end { + return Err(Error::CopyRead { + len: num_tag_bytes as u64, + src_len: src_end.offset_from(ip) as u64, + }); + } + + let loaded = if ip.add(4) <= src_end { + bytes::loadu_u32_le(ip) + } else { + let mut v = 0u32; + for i in 0..num_tag_bytes { + v |= (*ip.add(i) as u32) << (i * 8); + } + v + }; + let extracted = + (loaded & extract_offset_mask(tag_type)) as usize; + let offset = (entry_val & 0x700) | extracted; + ip = ip.add(num_tag_bytes); + + let srcp = op.sub(offset); + if (srcp as usize) < dst_base_addr || offset == 0 { + self.s = ip.offset_from(src) as usize; + self.d = op.offset_from(dst_base) as usize; + return Err(Error::Offset { + offset: offset as u64, + dst_pos: self.d as u64, + }); + } + + let end = op.add(len); + if end > dst_end { + self.s = ip.offset_from(src) as usize; + self.d = op.offset_from(dst_base) as usize; + return Err(Error::CopyWrite { + len: len as u64, + dst_len: (self.dst.len() - self.d) as u64, + }); + } + + if end.add(24) <= dst_end { + copy_dispatch(op, offset, len); + } else { + let mut p = op; + while p < end { + *p = *p.sub(offset); + p = p.add(1); + } + } + op = end; + } } + self.s = ip.offset_from(src) as usize; + self.d = op.offset_from(dst_base) as usize; } if self.d != self.dst.len() { return Err(Error::HeaderMismatch { @@ -386,74 +460,6 @@ impl<'s, 'd> Decompress<'s, 'd> { self.d += len as usize; Ok(()) } - - /// Reads a copy tag and writes the decompressed bytes. - #[inline(always)] - fn read_copy(&mut self, tag_byte: u8) -> Result<()> { - let entry_val = TAG_LOOKUP_TABLE[tag_byte as usize] as usize; - let tag_type = (tag_byte & 3) as usize; - let num_tag_bytes = tag_type + (tag_type == 3) as usize; - let len = entry_val & 0xFF; - - // Read offset from compressed input. - let trailer = if self.s + 4 <= self.src.len() { - unsafe { - let p = self.src.as_ptr().add(self.s); - let mask = u32::MAX >> ((4 - num_tag_bytes as u32) << 3); - bytes::loadu_u32_le(p) as usize & mask as usize - } - } else if num_tag_bytes == 1 { - if self.s >= self.src.len() { - return Err(Error::CopyRead { - len: 1, - src_len: (self.src.len() - self.s) as u64, - }); - } - self.src[self.s] as usize - } else if num_tag_bytes == 2 { - if self.s + 1 >= self.src.len() { - return Err(Error::CopyRead { - len: 2, - src_len: (self.src.len() - self.s) as u64, - }); - } - bytes::read_u16_le(&self.src[self.s..]) as usize - } else { - return Err(Error::CopyRead { - len: num_tag_bytes as u64, - src_len: (self.src.len() - self.s) as u64, - }); - }; - let offset = (entry_val & 0x700) | trailer; - self.s += num_tag_bytes; - - if self.d <= offset.wrapping_sub(1) { - return Err(Error::Offset { - offset: offset as u64, - dst_pos: self.d as u64, - }); - } - let end = self.d + len; - if end + 24 <= self.dst.len() { - unsafe { - copy_dispatch(self.dst.as_mut_ptr().add(self.d), offset, len); - } - } else { - if end > self.dst.len() { - return Err(Error::CopyWrite { - len: len as u64, - dst_len: (self.dst.len() - self.d) as u64, - }); - } - // Byte-by-byte fallback for the last few bytes. - while self.d != end { - self.dst[self.d] = self.dst[self.d - offset]; - self.d += 1; - } - } - self.d = end; - Ok(()) - } } /// Header represents the single varint that starts every Snappy compressed From 51591f5058861a70be3f71e9fea3eb74373113c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 12 Mar 2026 10:21:28 +0100 Subject: [PATCH 31/41] Remove --- src/bytes.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/bytes.rs b/src/bytes.rs index 4f198c6..b3e62c4 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -1,12 +1,6 @@ use std::convert::TryInto; use std::io; -/// Read a u16 in little endian format from the beginning of the given slice. -/// This panics if the slice has length less than 2. -pub fn read_u16_le(slice: &[u8]) -> u16 { - u16::from_le_bytes(slice[..2].try_into().unwrap()) -} - /// Read a u24 (returned as a u32 with the most significant 8 bits always set /// to 0) in little endian format from the beginning of the given slice. This /// panics if the slice has length less than 3. From 6b632c4de389884ab6daf5ad64fde4f9efb1be91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 12 Mar 2026 11:01:06 +0100 Subject: [PATCH 32/41] WIP --- src/decompress.rs | 141 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 115 insertions(+), 26 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 2d50c15..dbcb811 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -14,7 +14,7 @@ const TAG_LOOKUP_TABLE: [u16; 256] = tag::TAG_LOOKUP_TABLE; /// Copy up to 64 bytes using unrolled 16-byte copies. /// `src` and `dst` must not overlap in each 16-byte chunk. /// Use `wide_copy_long` when src and dst are guaranteed >= 32 apart. -#[inline(always)] +#[inline] unsafe fn wide_copy(src: *const u8, dst: *mut u8, len: usize) { debug_assert!(len <= 64); ptr::copy_nonoverlapping(src, dst, 16); @@ -57,6 +57,20 @@ fn extract_offset_mask(tag_type: usize) -> u32 { } } +/// Cold path for copy dispatch — handles len > 16 or offset < 8 cases. +/// Kept out-of-line on x86 to reduce i-cache pressure in the hot loop. +#[inline(never)] +unsafe fn copy_dispatch_cold(dst: *mut u8, offset: usize, len: usize) { + let srcp = dst.sub(offset); + if offset >= 32 { + wide_copy_long(srcp, dst, len); + } else if offset >= 16 { + wide_copy(srcp, dst, len); + } else { + overlapping_copy(dst, offset, len); + } +} + /// Dispatch a copy of `len` bytes from `dst - offset` to `dst`. /// /// Tries fast wide-copy paths (non-overlapping 8/16/32 byte chunks). @@ -64,18 +78,31 @@ fn extract_offset_mask(tag_type: usize) -> u32 { /// /// Caller must ensure at least `len + 24` bytes of writable space at `dst` /// and at least `offset` valid bytes preceding `dst`. +/// +/// On ARM: all paths are inlined (plenty of registers and i-cache). +/// On x86: only the hot path (len≤16, offset≥8) is inlined; the rest +/// goes through `copy_dispatch_cold` to keep the loop compact. #[inline(always)] unsafe fn copy_dispatch(dst: *mut u8, offset: usize, len: usize) { let srcp = dst.sub(offset); if len <= 16 && offset >= 8 { ptr::copy_nonoverlapping(srcp, dst, 8); ptr::copy_nonoverlapping(srcp.add(8), dst.add(8), 8); - } else if offset >= 32 { - wide_copy_long(srcp, dst, len); - } else if offset >= 16 { - wide_copy(srcp, dst, len); } else { - overlapping_copy(dst, offset, len); + #[cfg(target_arch = "aarch64")] + { + if offset >= 32 { + wide_copy_long(srcp, dst, len); + } else if offset >= 16 { + wide_copy(srcp, dst, len); + } else { + overlapping_copy(dst, offset, len); + } + } + #[cfg(not(target_arch = "aarch64"))] + { + copy_dispatch_cold(dst, offset, len); + } } } @@ -85,7 +112,7 @@ unsafe fn copy_dispatch(dst: *mut u8, offset: usize, len: usize) { /// /// Caller must ensure `dst + len + 24` is writable and that `dst` is /// preceded by at least `offset` valid bytes. -#[inline(always)] +#[inline(never)] unsafe fn overlapping_copy(dst: *mut u8, offset: usize, len: usize) { let end = dst.add(len); let mut dstp = dst; @@ -217,6 +244,10 @@ impl<'s, 'd> Decompress<'s, 'd> { /// This assumes that the header has already been read and that `dst` is /// big enough to store all decompressed bytes. fn decompress(&mut self) -> Result<()> { + // Fast inner loop: processes bulk data without bounds checks, + // with tag preloading. ARM: fully inlined copies, pointer-based + // offset check (ccmp fusion). x86: compact copies (cold paths + // out-of-line), integer offset check. unsafe { self.decompress_fast()?; } @@ -227,9 +258,9 @@ impl<'s, 'd> Decompress<'s, 'd> { let dst_base = self.dst.as_mut_ptr(); let src_end = src.add(self.src.len()); let dst_end = dst_base.add(self.dst.len()); - let dst_base_addr = dst_base as usize; let mut ip = src.add(self.s); let mut op = dst_base.add(self.d); + let mut d = self.d; while ip < src_end { let byte = *ip; @@ -238,10 +269,11 @@ impl<'s, 'd> Decompress<'s, 'd> { if byte & 3 == 0 { let len = (byte >> 2) as usize + 1; self.s = ip.offset_from(src) as usize; - self.d = op.offset_from(dst_base) as usize; + self.d = d; self.read_literal(len)?; ip = src.add(self.s); op = dst_base.add(self.d); + d = self.d; } else { let entry_val = TAG_LOOKUP_TABLE[byte as usize] as usize; let tag_type = (byte & 3) as usize; @@ -269,23 +301,22 @@ impl<'s, 'd> Decompress<'s, 'd> { let offset = (entry_val & 0x700) | extracted; ip = ip.add(num_tag_bytes); - let srcp = op.sub(offset); - if (srcp as usize) < dst_base_addr || offset == 0 { + if d <= offset.wrapping_sub(1) { self.s = ip.offset_from(src) as usize; - self.d = op.offset_from(dst_base) as usize; + self.d = d; return Err(Error::Offset { offset: offset as u64, - dst_pos: self.d as u64, + dst_pos: d as u64, }); } let end = op.add(len); if end > dst_end { self.s = ip.offset_from(src) as usize; - self.d = op.offset_from(dst_base) as usize; + self.d = d; return Err(Error::CopyWrite { len: len as u64, - dst_len: (self.dst.len() - self.d) as u64, + dst_len: (self.dst.len() - d) as u64, }); } @@ -299,10 +330,11 @@ impl<'s, 'd> Decompress<'s, 'd> { } } op = end; + d += len; } } self.s = ip.offset_from(src) as usize; - self.d = op.offset_from(dst_base) as usize; + self.d = d; } if self.d != self.dst.len() { return Err(Error::HeaderMismatch { @@ -315,6 +347,13 @@ impl<'s, 'd> Decompress<'s, 'd> { /// Fast inner loop using raw pointers. Requires 17 bytes of source /// headroom and 88 bytes of destination headroom to avoid bounds checks. + /// Uses tag preloading to avoid an extra memory load per iteration. + /// + /// On ARM: uses pointer-based offset check (enables `ccmp` fusion), + /// fully inlined copy helpers, 31 GPRs keep everything in registers. + /// On x86: uses integer `d` for offset check (saves a register), + /// compact copy dispatch (cold paths out-of-line to reduce i-cache + /// pressure). #[inline(always)] unsafe fn decompress_fast(&mut self) -> Result<()> { let src = self.src.as_ptr(); @@ -331,7 +370,13 @@ impl<'s, 'd> Decompress<'s, 'd> { let ip_limit = src.add(src_len - 17); let op_limit = dst_base.add(dst_len - 88); let src_end = src.add(src_len); + + // ARM: pointer comparison for offset check (enables ccmp fusion). + #[cfg(target_arch = "aarch64")] let dst_base_addr = dst_base as usize; + // x86: integer `d` for offset check (avoids extra pointer register). + #[cfg(not(target_arch = "aarch64"))] + let mut d = self.d; let mut preload = *ip as u32; @@ -352,18 +397,36 @@ impl<'s, 'd> Decompress<'s, 'd> { let offset = (entry_val & 0x700) | extracted; ip = ip.add(num_tag_bytes); - let srcp = op.sub(offset); - if (srcp as usize) < dst_base_addr || offset == 0 { - self.s = ip.offset_from(src) as usize; - self.d = op.offset_from(dst_base) as usize; - return Err(Error::Offset { - offset: offset as u64, - dst_pos: self.d as u64, - }); + #[cfg(target_arch = "aarch64")] + { + let srcp = op.sub(offset); + if (srcp as usize) < dst_base_addr || offset == 0 { + self.s = ip.offset_from(src) as usize; + self.d = op.offset_from(dst_base) as usize; + return Err(Error::Offset { + offset: offset as u64, + dst_pos: self.d as u64, + }); + } + } + #[cfg(not(target_arch = "aarch64"))] + { + if d <= offset.wrapping_sub(1) { + self.s = ip.offset_from(src) as usize; + self.d = d; + return Err(Error::Offset { + offset: offset as u64, + dst_pos: d as u64, + }); + } } copy_dispatch(op, offset, len); op = op.add(len); + #[cfg(not(target_arch = "aarch64"))] + { + d += len; + } preload = loaded >> (tag_type as u32 * 8); reload = tag_type == 3; @@ -374,18 +437,37 @@ impl<'s, 'd> Decompress<'s, 'd> { ptr::copy_nonoverlapping(ip, op, 16); ip = ip.add(len); op = op.add(len); + #[cfg(not(target_arch = "aarch64"))] + { + d += len; + } } else if len <= 60 && (ip as usize + len + 16) <= (src_end as usize) { wide_copy_long(ip, op, len); ip = ip.add(len); op = op.add(len); + #[cfg(not(target_arch = "aarch64"))] + { + d += len; + } } else { self.s = ip.offset_from(src) as usize; - self.d = op.offset_from(dst_base) as usize; + #[cfg(target_arch = "aarch64")] + { + self.d = op.offset_from(dst_base) as usize; + } + #[cfg(not(target_arch = "aarch64"))] + { + self.d = d; + } self.read_literal(len)?; ip = src.add(self.s); op = dst_base.add(self.d); + #[cfg(not(target_arch = "aarch64"))] + { + d = self.d; + } } } @@ -397,7 +479,14 @@ impl<'s, 'd> Decompress<'s, 'd> { } } self.s = ip.offset_from(src) as usize; - self.d = op.offset_from(dst_base) as usize; + #[cfg(target_arch = "aarch64")] + { + self.d = op.offset_from(dst_base) as usize; + } + #[cfg(not(target_arch = "aarch64"))] + { + self.d = d; + } Ok(()) } From 7cf7b8798fe461b9259e3356a452964663e59be4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 12 Mar 2026 11:24:14 +0100 Subject: [PATCH 33/41] Snap --- src/decompress.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index dbcb811..5dc8e66 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -382,7 +382,6 @@ impl<'s, 'd> Decompress<'s, 'd> { loop { let byte = preload as u8; - let mut reload = true; if byte & 3 != 0 { let entry_val = TAG_LOOKUP_TABLE[byte as usize] as usize; @@ -429,7 +428,15 @@ impl<'s, 'd> Decompress<'s, 'd> { } preload = loaded >> (tag_type as u32 * 8); - reload = tag_type == 3; + if ip > ip_limit || op > op_limit { + break; + } + // Copy-1/copy-2: next tag byte is already in preload. + // Copy-4: need to reload from memory. + if tag_type < 3 { + continue; + } + preload = *ip as u32; } else { let len = (byte >> 2) as usize + 1; ip = ip.add(1); @@ -469,12 +476,9 @@ impl<'s, 'd> Decompress<'s, 'd> { d = self.d; } } - } - - if ip > ip_limit || op > op_limit { - break; - } - if reload { + if ip > ip_limit || op > op_limit { + break; + } preload = *ip as u32; } } From abd1c04d501757407635b0d81d1c3d8e9384142c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 12 Mar 2026 11:38:02 +0100 Subject: [PATCH 34/41] Snap --- src/decompress.rs | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 5dc8e66..f271aa0 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -268,12 +268,24 @@ impl<'s, 'd> Decompress<'s, 'd> { if byte & 3 == 0 { let len = (byte >> 2) as usize + 1; - self.s = ip.offset_from(src) as usize; - self.d = d; - self.read_literal(len)?; - ip = src.add(self.s); - op = dst_base.add(self.d); - d = self.d; + // Inline short literals: single 16-byte copy avoids + // the overhead of syncing ip/op through self. + if len <= 16 + && ip.add(16) <= src_end + && op.add(16) <= dst_end + { + ptr::copy_nonoverlapping(ip, op, 16); + ip = ip.add(len); + op = op.add(len); + d += len; + } else { + self.s = ip.offset_from(src) as usize; + self.d = d; + self.read_literal(len)?; + ip = src.add(self.s); + op = dst_base.add(self.d); + d = self.d; + } } else { let entry_val = TAG_LOOKUP_TABLE[byte as usize] as usize; let tag_type = (byte & 3) as usize; @@ -322,7 +334,12 @@ impl<'s, 'd> Decompress<'s, 'd> { if end.add(24) <= dst_end { copy_dispatch(op, offset, len); + } else if offset >= len { + // Non-overlapping: safe to copy directly. + ptr::copy_nonoverlapping(op.sub(offset), op, len); } else { + // Overlapping (offset < len): forward byte-by-byte + // to correctly expand the repeated pattern. let mut p = op; while p < end { *p = *p.sub(offset); From 69a31eec0d13656b56fc06bf46c81262b032c2da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 12 Mar 2026 11:49:26 +0100 Subject: [PATCH 35/41] Snap --- src/decompress.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decompress.rs b/src/decompress.rs index f271aa0..a61aa22 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -332,7 +332,7 @@ impl<'s, 'd> Decompress<'s, 'd> { }); } - if end.add(24) <= dst_end { + if op.add(64) <= dst_end { copy_dispatch(op, offset, len); } else if offset >= len { // Non-overlapping: safe to copy directly. From e186004c9b1e12af4c9c096c4d6596648bf368ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 12 Mar 2026 12:27:07 +0100 Subject: [PATCH 36/41] Snap --- src/decompress.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index a61aa22..80be13d 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -332,7 +332,7 @@ impl<'s, 'd> Decompress<'s, 'd> { }); } - if op.add(64) <= dst_end { + if d + 64 <= self.dst.len() { copy_dispatch(op, offset, len); } else if offset >= len { // Non-overlapping: safe to copy directly. @@ -371,7 +371,9 @@ impl<'s, 'd> Decompress<'s, 'd> { /// On x86: uses integer `d` for offset check (saves a register), /// compact copy dispatch (cold paths out-of-line to reduce i-cache /// pressure). - #[inline(always)] + /// x86: out-of-line gives isolated register allocation (14 GPRs, no + /// spills). ARM: LLVM inlines regardless (31 GPRs, no pressure). + #[cfg_attr(not(target_arch = "aarch64"), inline(never))] unsafe fn decompress_fast(&mut self) -> Result<()> { let src = self.src.as_ptr(); let dst_base = self.dst.as_mut_ptr(); From 5f91a48a70d35e2f1f93e9b1e0dba5e24b8684c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 12 Mar 2026 12:35:41 +0100 Subject: [PATCH 37/41] Snap --- src/decompress.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decompress.rs b/src/decompress.rs index 80be13d..1f60cd2 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -468,7 +468,7 @@ impl<'s, 'd> Decompress<'s, 'd> { d += len; } } else if len <= 60 - && (ip as usize + len + 16) <= (src_end as usize) + && (ip as usize + 64) <= (src_end as usize) { wide_copy_long(ip, op, len); ip = ip.add(len); From eb05964b8082ca026bff3010346c6a7c39381ca0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 12 Mar 2026 13:10:42 +0100 Subject: [PATCH 38/41] Snap --- src/decompress.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 1f60cd2..b473b74 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -271,8 +271,8 @@ impl<'s, 'd> Decompress<'s, 'd> { // Inline short literals: single 16-byte copy avoids // the overhead of syncing ip/op through self. if len <= 16 - && ip.add(16) <= src_end - && op.add(16) <= dst_end + && (ip as usize + 16) <= (src_end as usize) + && d + 16 <= self.dst.len() { ptr::copy_nonoverlapping(ip, op, 16); ip = ip.add(len); @@ -292,14 +292,14 @@ impl<'s, 'd> Decompress<'s, 'd> { let num_tag_bytes = tag_type + (tag_type == 3) as usize; let len = entry_val & 0xFF; - if ip.add(num_tag_bytes) > src_end { + if (ip as usize + num_tag_bytes) > (src_end as usize) { return Err(Error::CopyRead { len: num_tag_bytes as u64, src_len: src_end.offset_from(ip) as u64, }); } - let loaded = if ip.add(4) <= src_end { + let loaded = if (ip as usize + 4) <= (src_end as usize) { bytes::loadu_u32_le(ip) } else { let mut v = 0u32; @@ -322,8 +322,7 @@ impl<'s, 'd> Decompress<'s, 'd> { }); } - let end = op.add(len); - if end > dst_end { + if d + len > self.dst.len() { self.s = ip.offset_from(src) as usize; self.d = d; return Err(Error::CopyWrite { @@ -340,13 +339,14 @@ impl<'s, 'd> Decompress<'s, 'd> { } else { // Overlapping (offset < len): forward byte-by-byte // to correctly expand the repeated pattern. + let end = op.add(len); let mut p = op; while p < end { *p = *p.sub(offset); p = p.add(1); } } - op = end; + op = op.add(len); d += len; } } @@ -467,8 +467,7 @@ impl<'s, 'd> Decompress<'s, 'd> { { d += len; } - } else if len <= 60 - && (ip as usize + 64) <= (src_end as usize) + } else if len <= 60 && (ip as usize + 64) <= (src_end as usize) { wide_copy_long(ip, op, len); ip = ip.add(len); From 12c553e811204b7eb866de61151c3a6845ad2020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Thu, 12 Mar 2026 13:32:33 +0100 Subject: [PATCH 39/41] Fix --- src/decompress.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/decompress.rs b/src/decompress.rs index b473b74..4c1585b 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -331,7 +331,7 @@ impl<'s, 'd> Decompress<'s, 'd> { }); } - if d + 64 <= self.dst.len() { + if d + 88 <= self.dst.len() { copy_dispatch(op, offset, len); } else if offset >= len { // Non-overlapping: safe to copy directly. From dfd83e561962057db9f5d2b4f7efc3007db93900 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Fri, 13 Mar 2026 07:30:20 +0100 Subject: [PATCH 40/41] Preload 2 --- src/decompress.rs | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index 4c1585b..ace1e52 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -398,22 +398,23 @@ impl<'s, 'd> Decompress<'s, 'd> { let mut d = self.d; let mut preload = *ip as u32; + let mut next_entry = TAG_LOOKUP_TABLE[preload as u8 as usize] as usize; + let mut next_loaded = bytes::loadu_u32_le(ip.add(1)); loop { let byte = preload as u8; if byte & 3 != 0 { - let entry_val = TAG_LOOKUP_TABLE[byte as usize] as usize; + let entry_val = next_entry; + let loaded = next_loaded; let tag_type = (byte & 3) as usize; let num_tag_bytes = tag_type + (tag_type == 3) as usize; let len = entry_val & 0xFF; - ip = ip.add(1); + ip = ip.add(1 + num_tag_bytes); - let loaded = bytes::loadu_u32_le(ip); let extracted = (loaded & extract_offset_mask(tag_type)) as usize; let offset = (entry_val & 0x700) | extracted; - ip = ip.add(num_tag_bytes); #[cfg(target_arch = "aarch64")] { @@ -439,6 +440,16 @@ impl<'s, 'd> Decompress<'s, 'd> { } } + // Compute next iteration's preload + start its + // table lookup and offset load early, so they + // execute in parallel with copy_dispatch stores. + preload = loaded >> (tag_type as u32 * 8); + if tag_type >= 3 { + preload = *ip as u32; + } + next_entry = TAG_LOOKUP_TABLE[preload as u8 as usize] as usize; + next_loaded = bytes::loadu_u32_le(ip.add(1)); + copy_dispatch(op, offset, len); op = op.add(len); #[cfg(not(target_arch = "aarch64"))] @@ -446,16 +457,9 @@ impl<'s, 'd> Decompress<'s, 'd> { d += len; } - preload = loaded >> (tag_type as u32 * 8); if ip > ip_limit || op > op_limit { break; } - // Copy-1/copy-2: next tag byte is already in preload. - // Copy-4: need to reload from memory. - if tag_type < 3 { - continue; - } - preload = *ip as u32; } else { let len = (byte >> 2) as usize + 1; ip = ip.add(1); @@ -498,6 +502,8 @@ impl<'s, 'd> Decompress<'s, 'd> { break; } preload = *ip as u32; + next_entry = TAG_LOOKUP_TABLE[preload as u8 as usize] as usize; + next_loaded = bytes::loadu_u32_le(ip.add(1)); } } self.s = ip.offset_from(src) as usize; From ba0978fc08326a7c8a309849d4081471118b15b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Fri, 13 Mar 2026 10:27:43 +0100 Subject: [PATCH 41/41] Improve --- src/decompress.rs | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/src/decompress.rs b/src/decompress.rs index ace1e52..22cf963 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -408,12 +408,22 @@ impl<'s, 'd> Decompress<'s, 'd> { let entry_val = next_entry; let loaded = next_loaded; let tag_type = (byte & 3) as usize; - let num_tag_bytes = tag_type + (tag_type == 3) as usize; + + // Copy-4 never occurs in compressor output. + // In the fast loop it always produces offset=0 + // (error), so bail to the slow tail directly. + if tag_type == 3 { + break; + } + let len = entry_val & 0xFF; - ip = ip.add(1 + num_tag_bytes); + // tag_type is 1 or 2, so num_tag_bytes = tag_type. + ip = ip.add(1 + tag_type); - let extracted = - (loaded & extract_offset_mask(tag_type)) as usize; + // Shift-based mask: tag_type is guaranteed 1 or 2, + // so (1 << 8)-1 = 0xFF or (1 << 16)-1 = 0xFFFF. + let mask = (1u32 << (tag_type as u32 * 8)).wrapping_sub(1); + let extracted = (loaded & mask) as usize; let offset = (entry_val & 0x700) | extracted; #[cfg(target_arch = "aarch64")] @@ -440,13 +450,9 @@ impl<'s, 'd> Decompress<'s, 'd> { } } - // Compute next iteration's preload + start its - // table lookup and offset load early, so they - // execute in parallel with copy_dispatch stores. + // Preload next tag from the already-loaded data. + // tag_type is 1 or 2, so shift is 8 or 16. preload = loaded >> (tag_type as u32 * 8); - if tag_type >= 3 { - preload = *ip as u32; - } next_entry = TAG_LOOKUP_TABLE[preload as u8 as usize] as usize; next_loaded = bytes::loadu_u32_le(ip.add(1));