diff --git a/src/bytes.rs b/src/bytes.rs index 4f198c6..b3e62c4 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -1,12 +1,6 @@ use std::convert::TryInto; use std::io; -/// Read a u16 in little endian format from the beginning of the given slice. -/// This panics if the slice has length less than 2. -pub fn read_u16_le(slice: &[u8]) -> u16 { - u16::from_le_bytes(slice[..2].try_into().unwrap()) -} - /// Read a u24 (returned as a u32 with the most significant 8 bits always set /// to 0) in little endian format from the beginning of the given slice. This /// panics if the slice has length less than 3. diff --git a/src/decompress.rs b/src/decompress.rs index 2f4add2..22cf963 100644 --- a/src/decompress.rs +++ b/src/decompress.rs @@ -5,16 +5,132 @@ use crate::error::{Error, Result}; use crate::tag; use crate::MAX_INPUT_SIZE; -/// A lookup table for quickly computing the various attributes derived from a -/// tag byte. -const TAG_LOOKUP_TABLE: TagLookupTable = TagLookupTable(tag::TAG_LOOKUP_TABLE); +/// Tag lookup table. Keys are tag bytes, values are u16 with bit layout: +/// xxaa abbb xxcc cccc +/// Where `a` = num_tag_bytes (1/2/4), `b` = offset high bits (copy 1 only), +/// `c` = copy length (max 64). +const TAG_LOOKUP_TABLE: [u16; 256] = tag::TAG_LOOKUP_TABLE; -/// `WORD_MASK` is a map from the size of an integer in bytes to its -/// corresponding on a 32 bit integer. This is used when we need to read an -/// integer and we know there are at least 4 bytes to read from a buffer. In -/// this case, we can read a 32 bit little endian integer and mask out only the -/// bits we need. This in particular saves a branch. -const WORD_MASK: [usize; 5] = [0, 0xFF, 0xFFFF, 0xFFFFFF, 0xFFFFFFFF]; +/// Copy up to 64 bytes using unrolled 16-byte copies. +/// `src` and `dst` must not overlap in each 16-byte chunk. +/// Use `wide_copy_long` when src and dst are guaranteed >= 32 apart. +#[inline] +unsafe fn wide_copy(src: *const u8, dst: *mut u8, len: usize) { + debug_assert!(len <= 64); + ptr::copy_nonoverlapping(src, dst, 16); + ptr::copy_nonoverlapping(src.add(16), dst.add(16), 16); + if len > 32 { + ptr::copy_nonoverlapping(src.add(32), dst.add(32), 16); + if len > 48 { + ptr::copy_nonoverlapping(src.add(48), dst.add(48), 16); + } + } +} + +/// Copy up to 64 bytes using 32-byte copies (ldp/stp q pairs on ARM). +/// Requires src and dst to be at least 32 bytes apart (no overlap). +#[inline(always)] +unsafe fn wide_copy_long(src: *const u8, dst: *mut u8, len: usize) { + debug_assert!(len <= 64); + ptr::copy_nonoverlapping(src, dst, 32); + if len > 32 { + ptr::copy_nonoverlapping(src.add(32), dst.add(32), 32); + } +} + +/// Extract the offset mask for a given tag_type (1, 2, or 3). +/// Returns a mask: tag_type=1 → 0xFF, tag_type=2 → 0xFFFF, tag_type=3 → 0. +/// +/// On ARM, uses a packed u64 constant with shift (avoids memory load). +/// On x86, uses an array lookup (avoids 10-byte movabs + variable shift). +#[inline(always)] +fn extract_offset_mask(tag_type: usize) -> u32 { + #[cfg(target_arch = "aarch64")] + { + const MASKS_PACKED: u64 = 0x0000FFFF00FF0000u64; + ((MASKS_PACKED >> (tag_type * 16)) & 0xFFFF) as u32 + } + #[cfg(not(target_arch = "aarch64"))] + { + const MASKS: [u32; 4] = [0, 0xFF, 0xFFFF, 0]; + MASKS[tag_type] + } +} + +/// Cold path for copy dispatch — handles len > 16 or offset < 8 cases. +/// Kept out-of-line on x86 to reduce i-cache pressure in the hot loop. +#[inline(never)] +unsafe fn copy_dispatch_cold(dst: *mut u8, offset: usize, len: usize) { + let srcp = dst.sub(offset); + if offset >= 32 { + wide_copy_long(srcp, dst, len); + } else if offset >= 16 { + wide_copy(srcp, dst, len); + } else { + overlapping_copy(dst, offset, len); + } +} + +/// Dispatch a copy of `len` bytes from `dst - offset` to `dst`. +/// +/// Tries fast wide-copy paths (non-overlapping 8/16/32 byte chunks). +/// Falls back to `overlapping_copy` when offset < 16. +/// +/// Caller must ensure at least `len + 24` bytes of writable space at `dst` +/// and at least `offset` valid bytes preceding `dst`. +/// +/// On ARM: all paths are inlined (plenty of registers and i-cache). +/// On x86: only the hot path (len≤16, offset≥8) is inlined; the rest +/// goes through `copy_dispatch_cold` to keep the loop compact. +#[inline(always)] +unsafe fn copy_dispatch(dst: *mut u8, offset: usize, len: usize) { + let srcp = dst.sub(offset); + if len <= 16 && offset >= 8 { + ptr::copy_nonoverlapping(srcp, dst, 8); + ptr::copy_nonoverlapping(srcp.add(8), dst.add(8), 8); + } else { + #[cfg(target_arch = "aarch64")] + { + if offset >= 32 { + wide_copy_long(srcp, dst, len); + } else if offset >= 16 { + wide_copy(srcp, dst, len); + } else { + overlapping_copy(dst, offset, len); + } + } + #[cfg(not(target_arch = "aarch64"))] + { + copy_dispatch_cold(dst, offset, len); + } + } +} + +/// Copy `len` bytes from `dst - offset` into `dst`, handling overlapping +/// regions by expanding with `ptr::copy` until the gap >= 16, then +/// switching to non-overlapping 16-byte chunks. +/// +/// Caller must ensure `dst + len + 24` is writable and that `dst` is +/// preceded by at least `offset` valid bytes. +#[inline(never)] +unsafe fn overlapping_copy(dst: *mut u8, offset: usize, len: usize) { + let end = dst.add(len); + let mut dstp = dst; + let mut srcp = dst.sub(offset); + loop { + let diff = (dstp as usize) - (srcp as usize); + if diff >= 16 { + break; + } + ptr::copy(srcp, dstp, 16); + dstp = dstp.add(diff); + } + while dstp < end { + ptr::copy_nonoverlapping(srcp, dstp, 16); + srcp = srcp.add(16); + dstp = dstp.add(16); + } +} /// Returns the decompressed size (in bytes) of the compressed bytes given. /// @@ -128,15 +244,114 @@ impl<'s, 'd> Decompress<'s, 'd> { /// This assumes that the header has already been read and that `dst` is /// big enough to store all decompressed bytes. fn decompress(&mut self) -> Result<()> { - while self.s < self.src.len() { - let byte = self.src[self.s]; - self.s += 1; - if byte & 0b000000_11 == 0 { - let len = (byte >> 2) as usize + 1; - self.read_literal(len)?; - } else { - self.read_copy(byte)?; + // Fast inner loop: processes bulk data without bounds checks, + // with tag preloading. ARM: fully inlined copies, pointer-based + // offset check (ccmp fusion). x86: compact copies (cold paths + // out-of-line), integer offset check. + unsafe { + self.decompress_fast()?; + } + // Slow tail: process remaining bytes with bounds checks. + // Uses raw pointers like the fast path for consistency. + unsafe { + let src = self.src.as_ptr(); + let dst_base = self.dst.as_mut_ptr(); + let src_end = src.add(self.src.len()); + let dst_end = dst_base.add(self.dst.len()); + let mut ip = src.add(self.s); + let mut op = dst_base.add(self.d); + let mut d = self.d; + + while ip < src_end { + let byte = *ip; + ip = ip.add(1); + + if byte & 3 == 0 { + let len = (byte >> 2) as usize + 1; + // Inline short literals: single 16-byte copy avoids + // the overhead of syncing ip/op through self. + if len <= 16 + && (ip as usize + 16) <= (src_end as usize) + && d + 16 <= self.dst.len() + { + ptr::copy_nonoverlapping(ip, op, 16); + ip = ip.add(len); + op = op.add(len); + d += len; + } else { + self.s = ip.offset_from(src) as usize; + self.d = d; + self.read_literal(len)?; + ip = src.add(self.s); + op = dst_base.add(self.d); + d = self.d; + } + } else { + let entry_val = TAG_LOOKUP_TABLE[byte as usize] as usize; + let tag_type = (byte & 3) as usize; + let num_tag_bytes = tag_type + (tag_type == 3) as usize; + let len = entry_val & 0xFF; + + if (ip as usize + num_tag_bytes) > (src_end as usize) { + return Err(Error::CopyRead { + len: num_tag_bytes as u64, + src_len: src_end.offset_from(ip) as u64, + }); + } + + let loaded = if (ip as usize + 4) <= (src_end as usize) { + bytes::loadu_u32_le(ip) + } else { + let mut v = 0u32; + for i in 0..num_tag_bytes { + v |= (*ip.add(i) as u32) << (i * 8); + } + v + }; + let extracted = + (loaded & extract_offset_mask(tag_type)) as usize; + let offset = (entry_val & 0x700) | extracted; + ip = ip.add(num_tag_bytes); + + if d <= offset.wrapping_sub(1) { + self.s = ip.offset_from(src) as usize; + self.d = d; + return Err(Error::Offset { + offset: offset as u64, + dst_pos: d as u64, + }); + } + + if d + len > self.dst.len() { + self.s = ip.offset_from(src) as usize; + self.d = d; + return Err(Error::CopyWrite { + len: len as u64, + dst_len: (self.dst.len() - d) as u64, + }); + } + + if d + 88 <= self.dst.len() { + copy_dispatch(op, offset, len); + } else if offset >= len { + // Non-overlapping: safe to copy directly. + ptr::copy_nonoverlapping(op.sub(offset), op, len); + } else { + // Overlapping (offset < len): forward byte-by-byte + // to correctly expand the repeated pattern. + let end = op.add(len); + let mut p = op; + while p < end { + *p = *p.sub(offset); + p = p.add(1); + } + } + op = op.add(len); + d += len; + } } + self.s = ip.offset_from(src) as usize; + self.d = d; } if self.d != self.dst.len() { return Err(Error::HeaderMismatch { @@ -147,6 +362,168 @@ impl<'s, 'd> Decompress<'s, 'd> { Ok(()) } + /// Fast inner loop using raw pointers. Requires 17 bytes of source + /// headroom and 88 bytes of destination headroom to avoid bounds checks. + /// Uses tag preloading to avoid an extra memory load per iteration. + /// + /// On ARM: uses pointer-based offset check (enables `ccmp` fusion), + /// fully inlined copy helpers, 31 GPRs keep everything in registers. + /// On x86: uses integer `d` for offset check (saves a register), + /// compact copy dispatch (cold paths out-of-line to reduce i-cache + /// pressure). + /// x86: out-of-line gives isolated register allocation (14 GPRs, no + /// spills). ARM: LLVM inlines regardless (31 GPRs, no pressure). + #[cfg_attr(not(target_arch = "aarch64"), inline(never))] + unsafe fn decompress_fast(&mut self) -> Result<()> { + let src = self.src.as_ptr(); + let dst_base = self.dst.as_mut_ptr(); + let src_len = self.src.len(); + let dst_len = self.dst.len(); + + if src_len < 17 || dst_len < 88 { + return Ok(()); + } + + let mut ip = src.add(self.s); + let mut op = dst_base.add(self.d); + let ip_limit = src.add(src_len - 17); + let op_limit = dst_base.add(dst_len - 88); + let src_end = src.add(src_len); + + // ARM: pointer comparison for offset check (enables ccmp fusion). + #[cfg(target_arch = "aarch64")] + let dst_base_addr = dst_base as usize; + // x86: integer `d` for offset check (avoids extra pointer register). + #[cfg(not(target_arch = "aarch64"))] + let mut d = self.d; + + let mut preload = *ip as u32; + let mut next_entry = TAG_LOOKUP_TABLE[preload as u8 as usize] as usize; + let mut next_loaded = bytes::loadu_u32_le(ip.add(1)); + + loop { + let byte = preload as u8; + + if byte & 3 != 0 { + let entry_val = next_entry; + let loaded = next_loaded; + let tag_type = (byte & 3) as usize; + + // Copy-4 never occurs in compressor output. + // In the fast loop it always produces offset=0 + // (error), so bail to the slow tail directly. + if tag_type == 3 { + break; + } + + let len = entry_val & 0xFF; + // tag_type is 1 or 2, so num_tag_bytes = tag_type. + ip = ip.add(1 + tag_type); + + // Shift-based mask: tag_type is guaranteed 1 or 2, + // so (1 << 8)-1 = 0xFF or (1 << 16)-1 = 0xFFFF. + let mask = (1u32 << (tag_type as u32 * 8)).wrapping_sub(1); + let extracted = (loaded & mask) as usize; + let offset = (entry_val & 0x700) | extracted; + + #[cfg(target_arch = "aarch64")] + { + let srcp = op.sub(offset); + if (srcp as usize) < dst_base_addr || offset == 0 { + self.s = ip.offset_from(src) as usize; + self.d = op.offset_from(dst_base) as usize; + return Err(Error::Offset { + offset: offset as u64, + dst_pos: self.d as u64, + }); + } + } + #[cfg(not(target_arch = "aarch64"))] + { + if d <= offset.wrapping_sub(1) { + self.s = ip.offset_from(src) as usize; + self.d = d; + return Err(Error::Offset { + offset: offset as u64, + dst_pos: d as u64, + }); + } + } + + // Preload next tag from the already-loaded data. + // tag_type is 1 or 2, so shift is 8 or 16. + preload = loaded >> (tag_type as u32 * 8); + next_entry = TAG_LOOKUP_TABLE[preload as u8 as usize] as usize; + next_loaded = bytes::loadu_u32_le(ip.add(1)); + + copy_dispatch(op, offset, len); + op = op.add(len); + #[cfg(not(target_arch = "aarch64"))] + { + d += len; + } + + if ip > ip_limit || op > op_limit { + break; + } + } else { + let len = (byte >> 2) as usize + 1; + ip = ip.add(1); + if len <= 16 { + ptr::copy_nonoverlapping(ip, op, 16); + ip = ip.add(len); + op = op.add(len); + #[cfg(not(target_arch = "aarch64"))] + { + d += len; + } + } else if len <= 60 && (ip as usize + 64) <= (src_end as usize) + { + wide_copy_long(ip, op, len); + ip = ip.add(len); + op = op.add(len); + #[cfg(not(target_arch = "aarch64"))] + { + d += len; + } + } else { + self.s = ip.offset_from(src) as usize; + #[cfg(target_arch = "aarch64")] + { + self.d = op.offset_from(dst_base) as usize; + } + #[cfg(not(target_arch = "aarch64"))] + { + self.d = d; + } + self.read_literal(len)?; + ip = src.add(self.s); + op = dst_base.add(self.d); + #[cfg(not(target_arch = "aarch64"))] + { + d = self.d; + } + } + if ip > ip_limit || op > op_limit { + break; + } + preload = *ip as u32; + next_entry = TAG_LOOKUP_TABLE[preload as u8 as usize] as usize; + next_loaded = bytes::loadu_u32_le(ip.add(1)); + } + } + self.s = ip.offset_from(src) as usize; + #[cfg(target_arch = "aarch64")] + { + self.d = op.offset_from(dst_base) as usize; + } + #[cfg(not(target_arch = "aarch64"))] + { + self.d = d; + } + Ok(()) + } + /// Decompresses a literal from `src` starting at `s` to `dst` starting at /// `d` and returns the updated values of `s` and `d`. `s` should point to /// the byte immediately proceding the literal tag byte. @@ -161,34 +538,20 @@ impl<'s, 'd> Decompress<'s, 'd> { fn read_literal(&mut self, len: usize) -> Result<()> { debug_assert!(len <= 64); let mut len = len as u64; - // As an optimization for the common case, if the literal length is - // <=16 and we have enough room in both `src` and `dst`, copy the - // literal using unaligned loads and stores. - // - // We pick 16 bytes with the hope that it optimizes down to a 128 bit - // load/store. if len <= 16 && self.s + 16 <= self.src.len() && self.d + 16 <= self.dst.len() { unsafe { - // SAFETY: We know both src and dst have at least 16 bytes of - // wiggle room after s/d, even if `len` is <16, so the copy is - // safe. let srcp = self.src.as_ptr().add(self.s); let dstp = self.dst.as_mut_ptr().add(self.d); - // Hopefully uses SIMD registers for 128 bit load/store. ptr::copy_nonoverlapping(srcp, dstp, 16); } self.d += len as usize; self.s += len as usize; return Ok(()); } - // When the length is bigger than 60, it indicates that we need to read - // an additional 1-4 bytes to get the real length of the literal. if len >= 61 { - // If there aren't at least 4 bytes left to read then we know this - // is corrupt because the literal must have length >=61. if self.s as u64 + 4 > self.src.len() as u64 { return Err(Error::Literal { len: 4, @@ -196,16 +559,12 @@ impl<'s, 'd> Decompress<'s, 'd> { dst_len: (self.dst.len() - self.d) as u64, }); } - // Since we know there are 4 bytes left to read, read a 32 bit LE - // integer and mask away the bits we don't need. let byte_count = len as usize - 60; + let mask = u32::MAX >> ((4 - byte_count as u32) << 3); len = bytes::read_u32_le(&self.src[self.s..]) as u64; - len = (len & (WORD_MASK[byte_count] as u64)) + 1; + len = (len & mask as u64) + 1; self.s += byte_count; } - // If there's not enough buffer left to load or store this literal, - // then the input is corrupt. - // if self.s + len > self.src.len() || self.d + len > self.dst.len() { if ((self.src.len() - self.s) as u64) < len || ((self.dst.len() - self.d) as u64) < len { @@ -216,8 +575,6 @@ impl<'s, 'd> Decompress<'s, 'd> { }); } unsafe { - // SAFETY: We've already checked the bounds, so we know this copy - // is correct. let srcp = self.src.as_ptr().add(self.s); let dstp = self.dst.as_mut_ptr().add(self.d); ptr::copy_nonoverlapping(srcp, dstp, len as usize); @@ -226,121 +583,6 @@ impl<'s, 'd> Decompress<'s, 'd> { self.d += len as usize; Ok(()) } - - /// Reads a copy from `src` and writes the decompressed bytes to `dst`. `s` - /// should point to the byte immediately proceding the copy tag byte. - #[inline(always)] - fn read_copy(&mut self, tag_byte: u8) -> Result<()> { - // Find the copy offset and len, then advance the input past the copy. - // The rest of this function deals with reading/writing to output only. - let entry = TAG_LOOKUP_TABLE.entry(tag_byte); - let offset = entry.offset(self.src, self.s)?; - let len = entry.len(); - self.s += entry.num_tag_bytes(); - - // What we really care about here is whether `d == 0` or `d < offset`. - // To save an extra branch, use `d < offset - 1` instead. If `d` is - // `0`, then `offset.wrapping_sub(1)` will be usize::MAX which is also - // the max value of `d`. - if self.d <= offset.wrapping_sub(1) { - return Err(Error::Offset { - offset: offset as u64, - dst_pos: self.d as u64, - }); - } - // When all is said and done, dst is advanced to end. - let end = self.d + len; - // When the copy is small and the offset is at least 8 bytes away from - // `d`, then we can decompress the copy with two 64 bit unaligned - // loads/stores. - if offset >= 8 && len <= 16 && self.d + 16 <= self.dst.len() { - unsafe { - // SAFETY: We know dstp points to at least 16 bytes of memory - // from the condition above, and we also know that dstp is - // preceded by at least `offset` bytes from the `d <= offset` - // check above. - // - // We also know that dstp and dstp-8 do not overlap from the - // check above, justifying the use of copy_nonoverlapping. - let dstp = self.dst.as_mut_ptr().add(self.d); - let srcp = dstp.sub(offset); - // We can't do a single 16 byte load/store because src/dst may - // overlap with each other. Namely, the second copy here may - // copy bytes written in the first copy! - ptr::copy_nonoverlapping(srcp, dstp, 8); - ptr::copy_nonoverlapping(srcp.add(8), dstp.add(8), 8); - } - // If we have some wiggle room, try to decompress the copy 16 bytes - // at a time with 128 bit unaligned loads/stores. Remember, we can't - // just do a memcpy because decompressing copies may require copying - // overlapping memory. - // - // We need the extra wiggle room to make effective use of 128 bit - // loads/stores. Even if the store ends up copying more data than we - // need, we're careful to advance `d` by the correct amount at the end. - } else if end + 24 <= self.dst.len() { - unsafe { - // SAFETY: We know that dstp is preceded by at least `offset` - // bytes from the `d <= offset` check above. - // - // We don't know whether dstp overlaps with srcp, so we start - // by copying from srcp to dstp until they no longer overlap. - // The worst case is when dstp-src = 3 and copy length = 1. The - // first loop will issue these copy operations before stopping: - // - // [-1, 14] -> [0, 15] - // [-1, 14] -> [3, 18] - // [-1, 14] -> [9, 24] - // - // But the copy had length 1, so it was only supposed to write - // to [0, 0]. But the last copy wrote to [9, 24], which is 24 - // extra bytes in dst *beyond* the end of the copy, which is - // guaranteed by the conditional above. - - // Save destination length here to avoid a reborrow UB violation - // under the Tree Borrows model. - let dest_len = self.dst.len(); - - let mut dstp = self.dst.as_mut_ptr().add(self.d); - let mut srcp = dstp.sub(offset); - loop { - debug_assert!(dstp >= srcp); - let diff = (dstp as usize) - (srcp as usize); - if diff >= 16 { - break; - } - // srcp and dstp can overlap, so use ptr::copy. - debug_assert!(self.d + 16 <= dest_len); - ptr::copy(srcp, dstp, 16); - self.d += diff as usize; - dstp = dstp.add(diff); - } - while self.d < end { - ptr::copy_nonoverlapping(srcp, dstp, 16); - srcp = srcp.add(16); - dstp = dstp.add(16); - self.d += 16; - } - // At this point, `d` is likely wrong. We correct it before - // returning. It's correct value is `end`. - } - } else { - if end > self.dst.len() { - return Err(Error::CopyWrite { - len: len as u64, - dst_len: (self.dst.len() - self.d) as u64, - }); - } - // Finally, the slow byte-by-byte case, which should only be used - // for the last few bytes of decompression. - while self.d != end { - self.dst[self.d] = self.dst[self.d - offset]; - self.d += 1; - } - } - self.d = end; - Ok(()) - } } /// Header represents the single varint that starts every Snappy compressed @@ -373,103 +615,3 @@ impl Header { Ok(Header { len: header_len, decompress_len: decompress_len as usize }) } } - -/// A lookup table for quickly computing the various attributes derived from -/// a tag byte. The attributes are most useful for the three "copy" tags -/// and include the length of the copy, part of the offset (for copy 1-byte -/// only) and the total number of bytes proceding the tag byte that encode -/// the other part of the offset (1 for copy 1, 2 for copy 2 and 4 for copy 4). -/// -/// More specifically, the keys of the table are u8s and the values are u16s. -/// The bits of the values are laid out as follows: -/// -/// xxaa abbb xxcc cccc -/// -/// Where `a` is the number of bytes, `b` are the three bits of the offset -/// for copy 1 (the other 8 bits are in the byte proceding the tag byte; for -/// copy 2 and copy 4, `b = 0`), and `c` is the length of the copy (max of 64). -/// -/// We could pack this in fewer bits, but the position of the three `b` bits -/// lines up with the most significant three bits in the total offset for copy -/// 1, which avoids an extra shift instruction. -/// -/// In sum, this table is useful because it reduces branches and various -/// arithmetic operations. -struct TagLookupTable([u16; 256]); - -impl TagLookupTable { - /// Look up the tag entry given the tag `byte`. - #[inline(always)] - fn entry(&self, byte: u8) -> TagEntry { - TagEntry(self.0[byte as usize] as usize) - } -} - -/// Represents a single entry in the tag lookup table. -/// -/// See the documentation in `TagLookupTable` for the bit layout. -/// -/// The type is a `usize` for convenience. -struct TagEntry(usize); - -impl TagEntry { - /// Return the total number of bytes proceding this tag byte required to - /// encode the offset. - fn num_tag_bytes(&self) -> usize { - self.0 >> 11 - } - - /// Return the total copy length, capped at 255. - fn len(&self) -> usize { - self.0 & 0xFF - } - - /// Return the copy offset corresponding to this copy operation. `s` should - /// point to the position just after the tag byte that this entry was read - /// from. - /// - /// This requires reading from the compressed input since the offset is - /// encoded in bytes proceding the tag byte. - fn offset(&self, src: &[u8], s: usize) -> Result { - let num_tag_bytes = self.num_tag_bytes(); - let trailer = - // It is critical for this case to come first, since it is the - // fast path. We really hope that this case gets branch - // predicted. - if s + 4 <= src.len() { - unsafe { - // SAFETY: The conditional above guarantees that - // src[s..s+4] is valid to read from. - let p = src.as_ptr().add(s); - // We use WORD_MASK here to mask out the bits we don't - // need. While we're guaranteed to read 4 valid bytes, - // not all of those bytes are necessarily part of the - // offset. This is the key optimization: we don't need to - // branch on num_tag_bytes. - bytes::loadu_u32_le(p) as usize & WORD_MASK[num_tag_bytes] - } - } else if num_tag_bytes == 1 { - if s >= src.len() { - return Err(Error::CopyRead { - len: 1, - src_len: (src.len() - s) as u64, - }); - } - src[s] as usize - } else if num_tag_bytes == 2 { - if s + 1 >= src.len() { - return Err(Error::CopyRead { - len: 2, - src_len: (src.len() - s) as u64, - }); - } - bytes::read_u16_le(&src[s..]) as usize - } else { - return Err(Error::CopyRead { - len: num_tag_bytes as u64, - src_len: (src.len() - s) as u64, - }); - }; - Ok((self.0 & 0b0000_0111_0000_0000) | trailer) - } -}