From b2b726f5514fd18dc406318bda9a685f83c610f6 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Wed, 17 Jun 2026 16:00:38 -0400 Subject: [PATCH 01/11] feat(msgpack): add magic-ID prefix to all portable MessagePackCodec impls This is a proof-of-concept / starting point for making serialized sketch binaries self-describing. Every `to_msgpack()` output now starts with a single type-discriminant byte (analogous to Prometheus magic bytes for gauge/histogram), and `from_msgpack()` validates that byte before deserialising the payload. Magic-ID table (stable, never reuse a value): 0x01 HllSketch 0x02 CountMinSketch 0x03 CountMinSketchWithHeap 0x04 CountSketch 0x05 DdSketch 0x06 KllSketch 0x07 HydraKllSketch 0x08 SetAggregator 0x09 DeltaResult Wire format after this change: [ magic_id: u8 | ] API is unchanged: callers still call `to_msgpack()` / `from_msgpack()`. Round-trips still pass. The `BadMagicId` error variant surfaces mismatches at decode time instead of producing a confusing type error. All 426 unit tests pass. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- src/message_pack_format/error.rs | 14 ++++++ src/message_pack_format/magic_ids.rs | 43 +++++++++++++++++++ src/message_pack_format/mod.rs | 1 + .../portable/countminsketch.rs | 28 ++++++++---- .../portable/countminsketch_topk.rs | 17 ++++++-- .../portable/countsketch.rs | 14 ++++-- src/message_pack_format/portable/ddsketch.rs | 29 +++++++++---- .../portable/delta_set_aggregator.rs | 14 ++++-- src/message_pack_format/portable/hll.rs | 14 ++++-- src/message_pack_format/portable/hydra_kll.rs | 17 ++++++-- src/message_pack_format/portable/kll.rs | 17 ++++++-- .../portable/set_aggregator.rs | 20 +++++++-- 12 files changed, 189 insertions(+), 39 deletions(-) create mode 100644 src/message_pack_format/magic_ids.rs diff --git a/src/message_pack_format/error.rs b/src/message_pack_format/error.rs index a69452b..ef033cf 100644 --- a/src/message_pack_format/error.rs +++ b/src/message_pack_format/error.rs @@ -12,6 +12,9 @@ pub enum Error { Encode(rmp_serde::encode::Error), /// MessagePack decoding failed. Decode(rmp_serde::decode::Error), + /// The leading magic-ID byte was missing or did not match the expected value. + /// `got` is the byte that was found (or `None` if the buffer was empty). + BadMagicId { expected: u8, got: Option }, } impl fmt::Display for Error { @@ -19,6 +22,16 @@ impl fmt::Display for Error { match self { Error::Encode(e) => write!(f, "MessagePack encode failed: {e}"), Error::Decode(e) => write!(f, "MessagePack decode failed: {e}"), + Error::BadMagicId { expected, got } => match got { + Some(b) => write!( + f, + "MessagePack magic-ID mismatch: expected 0x{expected:02x}, got 0x{b:02x}" + ), + None => write!( + f, + "MessagePack magic-ID missing: expected 0x{expected:02x} but buffer is empty" + ), + }, } } } @@ -28,6 +41,7 @@ impl StdError for Error { match self { Error::Encode(e) => Some(e), Error::Decode(e) => Some(e), + Error::BadMagicId { .. } => None, } } } diff --git a/src/message_pack_format/magic_ids.rs b/src/message_pack_format/magic_ids.rs new file mode 100644 index 0000000..1afc446 --- /dev/null +++ b/src/message_pack_format/magic_ids.rs @@ -0,0 +1,43 @@ +//! Magic-ID constants for the portable MessagePack wire format. +//! +//! Every serialized binary produced by [`crate::message_pack_format::MessagePackCodec`] +//! is prefixed with a single byte that identifies the sketch type, analogous to +//! how Prometheus uses magic bytes to discriminate metric types. +//! +//! The prefix layout is: +//! +//! ```text +//! [ magic_id: u8 | ] +//! ``` +//! +//! Magic IDs are stable across versions. Adding a new sketch type requires a +//! new constant here; removing or repurposing an existing constant is a +//! breaking protocol change. The Go mirror of this table lives in +//! `sketchlib-go/wire/asapmsgpack/magic_ids.go`. + +/// HLL sketch (all variants: Regular, Datafusion, Hip). +pub const HLL: u8 = 0x01; + +/// Count-Min sketch (f64 counters, no top-k heap). +pub const COUNT_MIN_SKETCH: u8 = 0x02; + +/// Count-Min sketch with top-k heap (f64 counters + heap). +pub const COUNT_MIN_SKETCH_WITH_HEAP: u8 = 0x03; + +/// Count Sketch (signed f64 counters). +pub const COUNT_SKETCH: u8 = 0x04; + +/// DDSketch (quantile sketch, alpha-parameterised bucket array). +pub const DD_SKETCH: u8 = 0x05; + +/// KLL quantile sketch. +pub const KLL_SKETCH: u8 = 0x06; + +/// Hydra-KLL sketch (grid of KLL cells). +pub const HYDRA_KLL_SKETCH: u8 = 0x07; + +/// Set aggregator (distinct string set). +pub const SET_AGGREGATOR: u8 = 0x08; + +/// Delta-set aggregator result (added / removed string sets). +pub const DELTA_RESULT: u8 = 0x09; diff --git a/src/message_pack_format/mod.rs b/src/message_pack_format/mod.rs index 7734463..3f16d9f 100644 --- a/src/message_pack_format/mod.rs +++ b/src/message_pack_format/mod.rs @@ -20,6 +20,7 @@ pub mod codec; pub mod error; +pub mod magic_ids; pub mod native; pub mod portable; diff --git a/src/message_pack_format/portable/countminsketch.rs b/src/message_pack_format/portable/countminsketch.rs index 41fd91a..1eafed7 100644 --- a/src/message_pack_format/portable/countminsketch.rs +++ b/src/message_pack_format/portable/countminsketch.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; +use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec, magic_ids}; use crate::sketches::countminsketch::CountMin; use crate::{DataInput, FastPath, Vector2D}; @@ -442,17 +442,27 @@ impl MessagePackCodec for CountMinSketch { rows: self.rows, cols: self.cols, }; - Ok(rmp_serde::to_vec(&wire)?) + let mut out = vec![magic_ids::COUNT_MIN_SKETCH]; + out.extend(rmp_serde::to_vec(&wire)?); + Ok(out) } fn from_msgpack(bytes: &[u8]) -> Result { - let wire: CountMinSketchWire = rmp_serde::from_slice(bytes)?; - let backend = sketchlib_cms_from_matrix(wire.rows, wire.cols, &wire.sketch); - Ok(Self { - rows: wire.rows, - cols: wire.cols, - backend, - }) + match bytes.first() { + Some(&magic_ids::COUNT_MIN_SKETCH) => { + let wire: CountMinSketchWire = rmp_serde::from_slice(&bytes[1..])?; + let backend = sketchlib_cms_from_matrix(wire.rows, wire.cols, &wire.sketch); + Ok(Self { + rows: wire.rows, + cols: wire.cols, + backend, + }) + } + other => Err(MsgPackError::BadMagicId { + expected: magic_ids::COUNT_MIN_SKETCH, + got: other.copied(), + }), + } } } diff --git a/src/message_pack_format/portable/countminsketch_topk.rs b/src/message_pack_format/portable/countminsketch_topk.rs index fa850f9..32d5431 100644 --- a/src/message_pack_format/portable/countminsketch_topk.rs +++ b/src/message_pack_format/portable/countminsketch_topk.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; +use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec, magic_ids}; use crate::sketches::countminsketch_topk::CMSHeap; use crate::{DataInput, RegularPath, Vector2D}; @@ -297,11 +297,22 @@ impl MessagePackCodec for CountMinSketchWithHeap { topk_heap: self.topk_heap_items(), heap_size: self.heap_size, }; - Ok(rmp_serde::to_vec(&wire)?) + let mut out = vec![magic_ids::COUNT_MIN_SKETCH_WITH_HEAP]; + out.extend(rmp_serde::to_vec(&wire)?); + Ok(out) } fn from_msgpack(bytes: &[u8]) -> Result { - let wire: CountMinSketchWithHeapWire = rmp_serde::from_slice(bytes)?; + let payload = match bytes.first() { + Some(&magic_ids::COUNT_MIN_SKETCH_WITH_HEAP) => &bytes[1..], + other => { + return Err(MsgPackError::BadMagicId { + expected: magic_ids::COUNT_MIN_SKETCH_WITH_HEAP, + got: other.copied(), + }) + } + }; + let wire: CountMinSketchWithHeapWire = rmp_serde::from_slice(payload)?; let mut sorted_topk_heap = wire.topk_heap; sorted_topk_heap.sort_by(|a, b| b.value.partial_cmp(&a.value).unwrap()); diff --git a/src/message_pack_format/portable/countsketch.rs b/src/message_pack_format/portable/countsketch.rs index 408c693..a57a96c 100644 --- a/src/message_pack_format/portable/countsketch.rs +++ b/src/message_pack_format/portable/countsketch.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; +use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec, magic_ids}; // ===================================================================== // ASAP runtime wire-format-aligned variant . @@ -434,11 +434,19 @@ impl CountSketch { impl MessagePackCodec for CountSketch { fn to_msgpack(&self) -> Result, MsgPackError> { - Ok(rmp_serde::to_vec(self)?) + let mut out = vec![magic_ids::COUNT_SKETCH]; + out.extend(rmp_serde::to_vec(self)?); + Ok(out) } fn from_msgpack(bytes: &[u8]) -> Result { - Ok(rmp_serde::from_slice(bytes)?) + match bytes.first() { + Some(&magic_ids::COUNT_SKETCH) => Ok(rmp_serde::from_slice(&bytes[1..])?), + other => Err(MsgPackError::BadMagicId { + expected: magic_ids::COUNT_SKETCH, + got: other.copied(), + }), + } } } diff --git a/src/message_pack_format/portable/ddsketch.rs b/src/message_pack_format/portable/ddsketch.rs index 49414f3..bc0b425 100644 --- a/src/message_pack_format/portable/ddsketch.rs +++ b/src/message_pack_format/portable/ddsketch.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; +use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec, magic_ids}; /// Bucket-store growth chunk for the wire-format-aligned [`DdSketch`] /// variant. Matches the Go reference implementation's `DDSketch.GrowChunk` @@ -403,11 +403,19 @@ impl DdSketch { impl MessagePackCodec for DdSketch { fn to_msgpack(&self) -> Result, MsgPackError> { - Ok(rmp_serde::to_vec(self)?) + let mut out = vec![magic_ids::DD_SKETCH]; + out.extend(rmp_serde::to_vec(self)?); + Ok(out) } fn from_msgpack(bytes: &[u8]) -> Result { - Ok(rmp_serde::from_slice(bytes)?) + match bytes.first() { + Some(&magic_ids::DD_SKETCH) => Ok(rmp_serde::from_slice(&bytes[1..])?), + other => Err(MsgPackError::BadMagicId { + expected: magic_ids::DD_SKETCH, + got: other.copied(), + }), + } } } @@ -539,16 +547,21 @@ mod tests { /// DataPoint-level METRIC scalars (count/sum/min/max). This pins the /// element count so the bytes stay parity-aligned with the Go /// reference implementation. + /// + /// bytes[0] is now the magic-ID prefix (0x05 = DD_SKETCH); the msgpack + /// payload starts at bytes[1]. #[test] fn test_msgpack_is_three_element_array() { + use crate::message_pack_format::magic_ids; let sk = DdSketch::from_raw(0.01, vec![1, 2, 3], -2); let bytes = sk.to_msgpack().unwrap(); - // rmp compact encoding leads with an array marker. A fixarray of - // length 3 is the single byte 0x93 (0b1001_0011). + assert_eq!(bytes[0], magic_ids::DD_SKETCH, "expected magic-ID byte"); + // rmp compact encoding of the payload leads with an array marker. + // A fixarray of length 3 is the single byte 0x93 (0b1001_0011). assert_eq!( - bytes[0], 0x93, - "expected a 3-element msgpack fixarray (0x93), got {:#04x}", - bytes[0] + bytes[1], 0x93, + "expected a 3-element msgpack fixarray (0x93) at bytes[1], got {:#04x}", + bytes[1] ); } diff --git a/src/message_pack_format/portable/delta_set_aggregator.rs b/src/message_pack_format/portable/delta_set_aggregator.rs index d26dbeb..6a87dda 100644 --- a/src/message_pack_format/portable/delta_set_aggregator.rs +++ b/src/message_pack_format/portable/delta_set_aggregator.rs @@ -8,7 +8,7 @@ use std::collections::HashSet; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; +use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec, magic_ids}; /// Wire DTO for the delta set aggregator: a snapshot of added/removed /// string keys between two consecutive observations. @@ -20,11 +20,19 @@ pub struct DeltaResult { impl MessagePackCodec for DeltaResult { fn to_msgpack(&self) -> Result, MsgPackError> { - Ok(rmp_serde::to_vec(self)?) + let mut out = vec![magic_ids::DELTA_RESULT]; + out.extend(rmp_serde::to_vec(self)?); + Ok(out) } fn from_msgpack(bytes: &[u8]) -> Result { - Ok(rmp_serde::from_slice(bytes)?) + match bytes.first() { + Some(&magic_ids::DELTA_RESULT) => Ok(rmp_serde::from_slice(&bytes[1..])?), + other => Err(MsgPackError::BadMagicId { + expected: magic_ids::DELTA_RESULT, + got: other.copied(), + }), + } } } diff --git a/src/message_pack_format/portable/hll.rs b/src/message_pack_format/portable/hll.rs index 17b8686..d87e7ab 100644 --- a/src/message_pack_format/portable/hll.rs +++ b/src/message_pack_format/portable/hll.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; +use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec, magic_ids}; use crate::{CANONICAL_HASH_SEED, DataInput, hash64_seeded}; /// HLL estimator variant. Mirrors `asap_sketchlib::proto::sketchlib::HllVariant`. @@ -430,11 +430,19 @@ fn read_uvarint(buf: &[u8]) -> Option<(u64, usize)> { impl MessagePackCodec for HllSketch { fn to_msgpack(&self) -> Result, MsgPackError> { - Ok(rmp_serde::to_vec(self)?) + let mut out = vec![magic_ids::HLL]; + out.extend(rmp_serde::to_vec(self)?); + Ok(out) } fn from_msgpack(bytes: &[u8]) -> Result { - Ok(rmp_serde::from_slice(bytes)?) + match bytes.first() { + Some(&magic_ids::HLL) => Ok(rmp_serde::from_slice(&bytes[1..])?), + other => Err(MsgPackError::BadMagicId { + expected: magic_ids::HLL, + got: other.copied(), + }), + } } } diff --git a/src/message_pack_format/portable/hydra_kll.rs b/src/message_pack_format/portable/hydra_kll.rs index b2d57e8..5e64183 100644 --- a/src/message_pack_format/portable/hydra_kll.rs +++ b/src/message_pack_format/portable/hydra_kll.rs @@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize}; use xxhash_rust::xxh32::xxh32; use crate::message_pack_format::portable::kll::{KllSketch, KllSketchData}; -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; +use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec, magic_ids}; #[derive(Debug, Clone)] pub struct HydraKllSketch { @@ -157,14 +157,25 @@ impl MessagePackCodec for HydraKllSketch { cols: self.cols, sketches, }; - Ok(rmp_serde::to_vec(&wire)?) + let mut out = vec![magic_ids::HYDRA_KLL_SKETCH]; + out.extend(rmp_serde::to_vec(&wire)?); + Ok(out) } fn from_msgpack(bytes: &[u8]) -> Result { use crate::sketches::kll::KLL; use rmp_serde::decode::Error as RmpDecodeError; - let wire: HydraKllSketchWire = rmp_serde::from_slice(bytes)?; + let payload = match bytes.first() { + Some(&magic_ids::HYDRA_KLL_SKETCH) => &bytes[1..], + other => { + return Err(MsgPackError::BadMagicId { + expected: magic_ids::HYDRA_KLL_SKETCH, + got: other.copied(), + }) + } + }; + let wire: HydraKllSketchWire = rmp_serde::from_slice(payload)?; if wire.sketches.len() != wire.rows { return Err(MsgPackError::Decode(RmpDecodeError::Uncategorized( diff --git a/src/message_pack_format/portable/kll.rs b/src/message_pack_format/portable/kll.rs index 55f5cf9..5b2aeca 100644 --- a/src/message_pack_format/portable/kll.rs +++ b/src/message_pack_format/portable/kll.rs @@ -9,7 +9,7 @@ use serde::{Deserialize, Serialize}; -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; +use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec, magic_ids}; use crate::sketches::kll::KLL; /// Concrete KLL type backing the wire-format `KllSketch`. @@ -347,11 +347,22 @@ impl MessagePackCodec for KllSketch { k: self.k, sketch_bytes: self.sketch_bytes(), }; - Ok(rmp_serde::to_vec(&wire)?) + let mut out = vec![magic_ids::KLL_SKETCH]; + out.extend(rmp_serde::to_vec(&wire)?); + Ok(out) } fn from_msgpack(bytes: &[u8]) -> Result { - let wire: KllSketchData = rmp_serde::from_slice(bytes)?; + let payload = match bytes.first() { + Some(&magic_ids::KLL_SKETCH) => &bytes[1..], + other => { + return Err(MsgPackError::BadMagicId { + expected: magic_ids::KLL_SKETCH, + got: other.copied(), + }) + } + }; + let wire: KllSketchData = rmp_serde::from_slice(payload)?; let backend = KLL::deserialize_from_bytes(&wire.sketch_bytes)?; Ok(Self { k: wire.k, backend }) } diff --git a/src/message_pack_format/portable/set_aggregator.rs b/src/message_pack_format/portable/set_aggregator.rs index d346a9e..a6131cd 100644 --- a/src/message_pack_format/portable/set_aggregator.rs +++ b/src/message_pack_format/portable/set_aggregator.rs @@ -9,7 +9,7 @@ use std::collections::HashSet; use serde::{Deserialize, Serialize}; -use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; +use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec, magic_ids}; /// Set aggregator for tracking a set of unique string keys. #[derive(Debug, Clone)] @@ -81,11 +81,22 @@ impl MessagePackCodec for SetAggregator { let wrapper = StringSetRef { values: &self.values, }; - Ok(rmp_serde::to_vec(&wrapper)?) + let mut out = vec![magic_ids::SET_AGGREGATOR]; + out.extend(rmp_serde::to_vec(&wrapper)?); + Ok(out) } fn from_msgpack(bytes: &[u8]) -> Result { - let wrapper: StringSetOwned = rmp_serde::from_slice(bytes)?; + let payload = match bytes.first() { + Some(&magic_ids::SET_AGGREGATOR) => &bytes[1..], + other => { + return Err(MsgPackError::BadMagicId { + expected: magic_ids::SET_AGGREGATOR, + got: other.copied(), + }) + } + }; + let wrapper: StringSetOwned = rmp_serde::from_slice(payload)?; Ok(Self { values: wrapper.values, }) @@ -154,8 +165,9 @@ mod tests { let mut sa = SetAggregator::new(); sa.update("a"); let bytes = sa.to_msgpack().unwrap(); + // Strip the leading magic byte before raw-decoding the payload. let decoded: StringSet = - rmp_serde::from_slice(&bytes).expect("should decode as StringSet { values: ... }"); + rmp_serde::from_slice(&bytes[1..]).expect("should decode as StringSet { values: ... }"); assert!(decoded.values.contains("a")); } } From 40821fa099d12ec39a7ddec3764caed19a1fee15 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Wed, 17 Jun 2026 16:15:11 -0400 Subject: [PATCH 02/11] feat(magic-id): extend to raw serialize_to_bytes/deserialize_from_bytes on all sketch types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All native sketch serialization methods now also carry a magic-ID prefix (range 0x81–0x89, separate from the portable 0x01–0x09 range): 0x81 CountMin<_, _> (native map-format) 0x82 Count<_, _, _> (native map-format) 0x83 CountL2HH / CMSHeap (native map-format) 0x84 HyperLogLogImpl<_,_,_> 0x85 HyperLogLogHIPImpl<_> 0x86 DDSketch (native map-format) 0x87 KLL 0x88 KLLDynamic 0x89 KMV The portable KllSketch and HydraKllSketch wire formats embed KLL bytes via KLL::serialize_to_bytes / deserialize_from_bytes; those call sites pick up the magic byte automatically and the portable round-trips continue to work. The native MessagePackCodec shims in message_pack_format/native/ delegate to these methods, so they too get the magic byte without any further changes. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- src/message_pack_format/magic_ids.rs | 36 +++++++++++++++++++++++++ src/sketches/countminsketch.rs | 21 ++++++++++++--- src/sketches/countsketch.rs | 21 ++++++++++++--- src/sketches/countsketch_topk.rs | 21 ++++++++++++--- src/sketches/ddsketch.rs | 26 +++++++++++++----- src/sketches/hll.rs | 40 ++++++++++++++++++++++------ src/sketches/kll.rs | 22 ++++++++++++--- src/sketches/kll_dynamic.rs | 27 ++++++++++++++----- src/sketches/kmv.rs | 19 ++++++++++--- 9 files changed, 192 insertions(+), 41 deletions(-) diff --git a/src/message_pack_format/magic_ids.rs b/src/message_pack_format/magic_ids.rs index 1afc446..2c1d2ad 100644 --- a/src/message_pack_format/magic_ids.rs +++ b/src/message_pack_format/magic_ids.rs @@ -41,3 +41,39 @@ pub const SET_AGGREGATOR: u8 = 0x08; /// Delta-set aggregator result (added / removed string sets). pub const DELTA_RESULT: u8 = 0x09; + +// ── Native (Rust-internal) sketch types ───────────────────────────────────── +// +// These are the generic sketch types in `crate::sketches`. Their wire format +// is produced by `serialize_to_bytes` / `deserialize_from_bytes` and is +// internal to Rust — Go (`sketchlib-go`) never reads these bytes directly. +// +// They use a separate range (0x81+) to make it visually clear that the byte +// refers to an internal format distinct from the portable cross-language ones. + +/// Generic Count-Min sketch (`sketches::CountMin<_, _>`). +pub const NATIVE_COUNT_MIN: u8 = 0x81; + +/// Generic Count Sketch (`sketches::Count<_, _, _>`). +pub const NATIVE_COUNT_SKETCH: u8 = 0x82; + +/// Count-Min + heavy-hitter heap (`sketches::CMSHeap` / `CountL2HH`). +pub const NATIVE_CMS_HEAP: u8 = 0x83; + +/// Generic HyperLogLog (`sketches::HyperLogLogImpl<_, _, _>` — Classic and ErtlMLE variants). +pub const NATIVE_HLL: u8 = 0x84; + +/// HyperLogLog HIP variant (`sketches::HyperLogLogHIPImpl<_>`). +pub const NATIVE_HLL_HIP: u8 = 0x85; + +/// DDSketch (`sketches::DDSketch`). +pub const NATIVE_DD_SKETCH: u8 = 0x86; + +/// KLL quantile sketch (`sketches::kll::KLL`). +pub const NATIVE_KLL: u8 = 0x87; + +/// Dynamic KLL quantile sketch (`sketches::kll_dynamic::KLLDynamic`). +pub const NATIVE_KLL_DYNAMIC: u8 = 0x88; + +/// KMV (K-Minimum Values) sketch (`sketches::KMV`). +pub const NATIVE_KMV: u8 = 0x89; diff --git a/src/sketches/countminsketch.rs b/src/sketches/countminsketch.rs index 43f296f..f68e8c3 100644 --- a/src/sketches/countminsketch.rs +++ b/src/sketches/countminsketch.rs @@ -262,16 +262,29 @@ impl CountMin { // Serialization helpers for CountMin. impl CountMin { - /// Serializes the sketch into MessagePack bytes. + /// Serializes the sketch into MessagePack bytes, prefixed with the + /// [`crate::message_pack_format::magic_ids::NATIVE_COUNT_MIN`] magic byte. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - to_vec_named(self) + let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_COUNT_MIN]; + out.extend(to_vec_named(self)?); + Ok(out) } } impl Deserialize<'de>, Mode, H: SketchHasher> CountMin { - /// Deserializes a sketch from MessagePack bytes. + /// Deserializes a sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - from_slice(bytes) + match bytes.first() { + Some(&crate::message_pack_format::magic_ids::NATIVE_COUNT_MIN) => { + from_slice(&bytes[1..]) + } + other => Err(RmpDecodeError::Uncategorized(format!( + "CountMin magic-ID mismatch: expected 0x{:02x}, got {:?}", + crate::message_pack_format::magic_ids::NATIVE_COUNT_MIN, + other.map(|b| format!("0x{b:02x}")) + .unwrap_or_else(|| "empty buffer".to_string()) + ))), + } } } diff --git a/src/sketches/countsketch.rs b/src/sketches/countsketch.rs index 4c5cd53..6609272 100644 --- a/src/sketches/countsketch.rs +++ b/src/sketches/countsketch.rs @@ -280,9 +280,12 @@ where S: MatrixStorage + Serialize, C: CountSketchCounter, { - /// Serializes the sketch into MessagePack bytes. + /// Serializes the sketch into MessagePack bytes, prefixed with the + /// [`crate::message_pack_format::magic_ids::NATIVE_COUNT_SKETCH`] magic byte. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - to_vec_named(self) + let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_COUNT_SKETCH]; + out.extend(to_vec_named(self)?); + Ok(out) } } @@ -292,9 +295,19 @@ where S: MatrixStorage + for<'de> Deserialize<'de>, C: CountSketchCounter, { - /// Deserializes a sketch from MessagePack bytes. + /// Deserializes a sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - from_slice(bytes) + match bytes.first() { + Some(&crate::message_pack_format::magic_ids::NATIVE_COUNT_SKETCH) => { + from_slice(&bytes[1..]) + } + other => Err(RmpDecodeError::Uncategorized(format!( + "Count magic-ID mismatch: expected 0x{:02x}, got {:?}", + crate::message_pack_format::magic_ids::NATIVE_COUNT_SKETCH, + other.map(|b| format!("0x{b:02x}")) + .unwrap_or_else(|| "empty buffer".to_string()) + ))), + } } } diff --git a/src/sketches/countsketch_topk.rs b/src/sketches/countsketch_topk.rs index c37e913..f19b51f 100644 --- a/src/sketches/countsketch_topk.rs +++ b/src/sketches/countsketch_topk.rs @@ -1106,14 +1106,27 @@ impl CountL2HH { compute_median_inline_f64(&mut lst[..]) } - /// Serializes the CountL2HH sketch into MessagePack bytes. + /// Serializes the CountL2HH sketch into MessagePack bytes, prefixed with the + /// [`crate::message_pack_format::magic_ids::NATIVE_CMS_HEAP`] magic byte. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - to_vec_named(self) + let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_CMS_HEAP]; + out.extend(to_vec_named(self)?); + Ok(out) } - /// Deserializes a CountL2HH sketch from MessagePack bytes. + /// Deserializes a CountL2HH sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - from_slice(bytes) + match bytes.first() { + Some(&crate::message_pack_format::magic_ids::NATIVE_CMS_HEAP) => { + from_slice(&bytes[1..]) + } + other => Err(RmpDecodeError::Uncategorized(format!( + "CountL2HH magic-ID mismatch: expected 0x{:02x}, got {:?}", + crate::message_pack_format::magic_ids::NATIVE_CMS_HEAP, + other.map(|b| format!("0x{b:02x}")) + .unwrap_or_else(|| "empty buffer".to_string()) + ))), + } } } diff --git a/src/sketches/ddsketch.rs b/src/sketches/ddsketch.rs index c213275..5669836 100644 --- a/src/sketches/ddsketch.rs +++ b/src/sketches/ddsketch.rs @@ -165,12 +165,16 @@ impl DDSketch { } } - /// Serializes the sketch to a MessagePack byte vector. + /// Serializes the sketch to a MessagePack byte vector, prefixed with the + /// [`crate::message_pack_format::magic_ids::NATIVE_DD_SKETCH`] magic byte. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - to_vec_named(self) + let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_DD_SKETCH]; + out.extend(to_vec_named(self)?); + Ok(out) } - /// Deserializes a DDSketch from a MessagePack byte slice. + /// Deserializes a DDSketch from a MessagePack byte slice produced by + /// [`Self::serialize_to_bytes`]. /// /// The `count`/`sum`/`min`/`max` scalars are `#[serde(skip)]` (dropped /// from the wire, ProjectASAP/sketchlib-go#243), so they default to @@ -179,9 +183,19 @@ impl DDSketch { /// reconstructed from the per-bucket representative values, accurate /// to within the sketch's α relative-accuracy bound. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - let mut sk: Self = from_slice(bytes)?; - sk.recompute_scalars_from_store(); - Ok(sk) + match bytes.first() { + Some(&crate::message_pack_format::magic_ids::NATIVE_DD_SKETCH) => { + let mut sk: Self = from_slice(&bytes[1..])?; + sk.recompute_scalars_from_store(); + Ok(sk) + } + other => Err(RmpDecodeError::Uncategorized(format!( + "DDSketch magic-ID mismatch: expected 0x{:02x}, got {:?}", + crate::message_pack_format::magic_ids::NATIVE_DD_SKETCH, + other.map(|b| format!("0x{b:02x}")) + .unwrap_or_else(|| "empty buffer".to_string()) + ))), + } } /// Rebuild the in-memory `count`/`sum`/`min`/`max` aggregates from the diff --git a/src/sketches/hll.rs b/src/sketches/hll.rs index dcdbae3..59bafa7 100644 --- a/src/sketches/hll.rs +++ b/src/sketches/hll.rs @@ -116,14 +116,25 @@ impl } } - /// Serializes the sketch into MessagePack bytes. + /// Serializes the sketch into MessagePack bytes, prefixed with the + /// [`crate::message_pack_format::magic_ids::NATIVE_HLL`] magic byte. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - to_vec_named(self) + let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_HLL]; + out.extend(to_vec_named(self)?); + Ok(out) } - /// Deserializes a sketch from MessagePack bytes. + /// Deserializes a sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - from_slice(bytes) + match bytes.first() { + Some(&crate::message_pack_format::magic_ids::NATIVE_HLL) => from_slice(&bytes[1..]), + other => Err(RmpDecodeError::Uncategorized(format!( + "HyperLogLogImpl magic-ID mismatch: expected 0x{:02x}, got {:?}", + crate::message_pack_format::magic_ids::NATIVE_HLL, + other.map(|b| format!("0x{b:02x}")) + .unwrap_or_else(|| "empty buffer".to_string()) + ))), + } } /// Borrow the raw register byte slice (one byte per register). @@ -369,14 +380,27 @@ impl HyperLogLogHIPImpl { self.est as usize } - /// Serializes the sketch into MessagePack bytes. + /// Serializes the sketch into MessagePack bytes, prefixed with the + /// [`crate::message_pack_format::magic_ids::NATIVE_HLL_HIP`] magic byte. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - to_vec_named(self) + let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_HLL_HIP]; + out.extend(to_vec_named(self)?); + Ok(out) } - /// Deserializes a sketch from MessagePack bytes. + /// Deserializes a sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - from_slice(bytes) + match bytes.first() { + Some(&crate::message_pack_format::magic_ids::NATIVE_HLL_HIP) => { + from_slice(&bytes[1..]) + } + other => Err(RmpDecodeError::Uncategorized(format!( + "HyperLogLogHIPImpl magic-ID mismatch: expected 0x{:02x}, got {:?}", + crate::message_pack_format::magic_ids::NATIVE_HLL_HIP, + other.map(|b| format!("0x{b:02x}")) + .unwrap_or_else(|| "empty buffer".to_string()) + ))), + } } } diff --git a/src/sketches/kll.rs b/src/sketches/kll.rs index ee11b3e..e060393 100644 --- a/src/sketches/kll.rs +++ b/src/sketches/kll.rs @@ -662,20 +662,34 @@ impl KLL { // -- Serialization ------------------------------------------------------- - /// Serializes the sketch to a MessagePack byte vector. + /// Serializes the sketch to a MessagePack byte vector, prefixed with the + /// [`crate::message_pack_format::magic_ids::NATIVE_KLL`] magic byte. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> where T: Serialize, { - rmp_serde::to_vec(self) + let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_KLL]; + out.extend(rmp_serde::to_vec(self)?); + Ok(out) } - /// Deserializes a KLL sketch from a MessagePack byte slice. + /// Deserializes a KLL sketch from a MessagePack byte slice produced by + /// [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result where T: for<'de> Deserialize<'de>, { - rmp_serde::from_slice(bytes) + match bytes.first() { + Some(&crate::message_pack_format::magic_ids::NATIVE_KLL) => { + rmp_serde::from_slice(&bytes[1..]) + } + other => Err(rmp_serde::decode::Error::Uncategorized(format!( + "KLL magic-ID mismatch: expected 0x{:02x}, got {:?}", + crate::message_pack_format::magic_ids::NATIVE_KLL, + other.map(|b| format!("0x{b:02x}")) + .unwrap_or_else(|| "empty buffer".to_string()) + ))), + } } fn ensure_levels_sorted(&mut self) { diff --git a/src/sketches/kll_dynamic.rs b/src/sketches/kll_dynamic.rs index e48f2eb..85a043f 100644 --- a/src/sketches/kll_dynamic.rs +++ b/src/sketches/kll_dynamic.rs @@ -353,23 +353,36 @@ impl KLLDynamic { self.items.len() } - /// Serialize the sketch into MessagePack bytes. + /// Serialize the sketch into MessagePack bytes, prefixed with the + /// [`crate::message_pack_format::magic_ids::NATIVE_KLL_DYNAMIC`] magic byte. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> where T: Serialize, { - rmp_serde::to_vec(self) + let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_KLL_DYNAMIC]; + out.extend(rmp_serde::to_vec(self)?); + Ok(out) } - /// Deserialize a sketch from MessagePack bytes. + /// Deserialize a sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result where T: for<'de> Deserialize<'de>, { - rmp_serde::from_slice(bytes).map(|mut sketch: KLLDynamic| { - sketch.rebuild_capacity_cache(); - sketch - }) + match bytes.first() { + Some(&crate::message_pack_format::magic_ids::NATIVE_KLL_DYNAMIC) => { + rmp_serde::from_slice(&bytes[1..]).map(|mut sketch: KLLDynamic| { + sketch.rebuild_capacity_cache(); + sketch + }) + } + other => Err(rmp_serde::decode::Error::Uncategorized(format!( + "KLLDynamic magic-ID mismatch: expected 0x{:02x}, got {:?}", + crate::message_pack_format::magic_ids::NATIVE_KLL_DYNAMIC, + other.map(|b| format!("0x{b:02x}")) + .unwrap_or_else(|| "empty buffer".to_string()) + ))), + } } } diff --git a/src/sketches/kmv.rs b/src/sketches/kmv.rs index 1517a14..d31cc27 100644 --- a/src/sketches/kmv.rs +++ b/src/sketches/kmv.rs @@ -77,14 +77,25 @@ impl KMV { } } - /// Serializes the sketch into MessagePack bytes. + /// Serializes the sketch into MessagePack bytes, prefixed with the + /// [`crate::message_pack_format::magic_ids::NATIVE_KMV`] magic byte. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - to_vec_named(self) + let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_KMV]; + out.extend(to_vec_named(self)?); + Ok(out) } - /// Deserializes a sketch from MessagePack bytes. + /// Deserializes a sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - from_slice(bytes) + match bytes.first() { + Some(&crate::message_pack_format::magic_ids::NATIVE_KMV) => from_slice(&bytes[1..]), + other => Err(RmpDecodeError::Uncategorized(format!( + "KMV magic-ID mismatch: expected 0x{:02x}, got {:?}", + crate::message_pack_format::magic_ids::NATIVE_KMV, + other.map(|b| format!("0x{b:02x}")) + .unwrap_or_else(|| "empty buffer".to_string()) + ))), + } } } From dad200f0a0402295d15c41ee732525010e480829 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Wed, 17 Jun 2026 16:19:41 -0400 Subject: [PATCH 03/11] chore: fix cargo fmt and add docs/msgpack-magic-ids.md - cargo fmt --all to fix formatting diffs caught by CI - Add docs/msgpack-magic-ids.md documenting the full magic-ID table, the portable vs native distinction, the embedded-KLL relationship, and instructions for adding future sketch types Co-Authored-By: Claude Sonnet 4.6 (1M context) --- docs/msgpack-magic-ids.md | 104 ++++++++++++++++++ .../portable/countminsketch_topk.rs | 2 +- src/message_pack_format/portable/hydra_kll.rs | 2 +- src/message_pack_format/portable/kll.rs | 2 +- .../portable/set_aggregator.rs | 2 +- src/sketches/countminsketch.rs | 3 +- src/sketches/countsketch.rs | 3 +- src/sketches/countsketch_topk.rs | 3 +- src/sketches/ddsketch.rs | 3 +- src/sketches/hll.rs | 10 +- src/sketches/kll.rs | 3 +- src/sketches/kll_dynamic.rs | 3 +- src/sketches/kmv.rs | 3 +- 13 files changed, 127 insertions(+), 16 deletions(-) create mode 100644 docs/msgpack-magic-ids.md diff --git a/docs/msgpack-magic-ids.md b/docs/msgpack-magic-ids.md new file mode 100644 index 0000000..f35fa49 --- /dev/null +++ b/docs/msgpack-magic-ids.md @@ -0,0 +1,104 @@ +# MessagePack Magic IDs + +Every serialised sketch binary produced by this library starts with a single +type-discriminant byte called the **magic ID**. Reading the first byte of any +blob is enough to identify what sketch type it contains — no full decode +required. + +``` +┌─────────────┬───────────────────────────────────────┐ +│ magic_id: u8 │ rmp_serde / msgpack payload … │ +└─────────────┴───────────────────────────────────────┘ +``` + +Magic IDs are **stable** — once assigned, a value is never reused or +reassigned. Adding a new sketch type requires a new constant; removing or +repurposing an existing constant is a **breaking protocol change**. + +The single source of truth in code is +[`src/message_pack_format/magic_ids.rs`](../src/message_pack_format/magic_ids.rs). +The Go mirror lives in +[`sketchlib-go/wire/asapmsgpack/magic_ids.go`](https://github.com/ProjectASAP/sketchlib-go/blob/main/wire/asapmsgpack/magic_ids.go). + +--- + +## Portable IDs (0x01 – 0x09) + +These IDs identify the **cross-language wire format** shared with +`sketchlib-go`. Any byte blob with a portable ID can be decoded by either +the Rust or Go implementation. + +| ID | Rust type / entry point | Go entry point | +|--------|--------------------------------------------------|----------------------------------------| +| `0x01` | `portable::HllSketch::to_msgpack` | `HLL.SerializeMsgpack` | +| `0x02` | `portable::CountMinSketch::to_msgpack` | `CountMinSketch.SerializeMsgpack` | +| `0x03` | `portable::CountMinSketchWithHeap::to_msgpack` | `CountSketch.SerializeMsgpackWithHeap` | +| `0x04` | `portable::CountSketch::to_msgpack` | `CountSketch.SerializeMsgpack` | +| `0x05` | `portable::DdSketch::to_msgpack` | `DDSketch.SerializeMsgpack` | +| `0x06` | `portable::KllSketch::to_msgpack` | _(Rust-only path; no Go equivalent)_ | +| `0x07` | `portable::HydraKllSketch::to_msgpack` | _(Rust-only path; no Go equivalent)_ | +| `0x08` | `portable::SetAggregator::to_msgpack` | _(Rust-only path; no Go equivalent)_ | +| `0x09` | `portable::DeltaResult::to_msgpack` | _(Rust-only path; no Go equivalent)_ | + +> **Note on `0x03`:** The Go producer calls this format "CountSketchWithHeap"; +> the Rust consumer knows it as `CountMinSketchWithHeap`. The delta-heap frame +> (`SerializeMsgpackWithHeapDelta`) shares the same magic ID because the Rust +> consumer uses the same `from_msgpack` path for both the full and delta shapes. + +--- + +## Native IDs (0x81 – 0x89) + +These IDs identify the **Rust-internal** format produced by +`serialize_to_bytes` / `deserialize_from_bytes` on the generic sketch types +in `src/sketches/`. Go never reads these bytes directly. + +The native wire format uses `rmp_serde`'s **named** (map) encoding +(`to_vec_named`), whereas the portable format uses **compact** (array) +encoding. The two are not interchangeable even for logically equivalent types +(e.g., `sketches::DDSketch` vs `portable::DdSketch`). + +| ID | Rust type / method | Notes | +|--------|--------------------------------------------------|-------| +| `0x81` | `sketches::CountMin::serialize_to_bytes` | Named map format; all `Mode` and `H` variants | +| `0x82` | `sketches::Count::serialize_to_bytes` | Named map format; all `Mode` and `H` variants | +| `0x83` | `sketches::CountL2HH::serialize_to_bytes` | Named map format (CMSHeap / heavy-hitter) | +| `0x84` | `sketches::HyperLogLogImpl::serialize_to_bytes` | Named map; Classic and ErtlMLE variants share one ID | +| `0x85` | `sketches::HyperLogLogHIPImpl::serialize_to_bytes` | Named map; HIP-specific accumulators | +| `0x86` | `sketches::DDSketch::serialize_to_bytes` | Named map; distinct from portable `0x05` | +| `0x87` | `sketches::KLL::serialize_to_bytes` | Compact array format (not named) | +| `0x88` | `sketches::KLLDynamic::serialize_to_bytes` | Compact array format (not named) | +| `0x89` | `sketches::KMV::serialize_to_bytes` | Named map format (experimental feature) | + +### Relationship between native and portable KLL + +`portable::KllSketch::to_msgpack` (`0x06`) embeds the raw KLL cell bytes +inside a `KllSketchData { k, sketch_bytes }` msgpack struct. Those embedded +bytes are produced by `KLL::serialize_to_bytes` and therefore carry the native +`0x87` prefix. The portable round-trip is: + +``` +KllSketch::to_msgpack() → [ 0x06 | msgpack([k, [0x87 | raw_kll_bytes]]) ] +KllSketch::from_msgpack() → strips 0x06, decodes struct, + calls KLL::deserialize_from_bytes([0x87 | …]) + → strips 0x87, decodes KLL +``` + +The same pattern applies to `HydraKllSketch` (`0x07`), which contains a grid +of KLL cells. + +--- + +## Adding a new sketch type + +1. Add a constant to `src/message_pack_format/magic_ids.rs` (choose the next + available value in the appropriate range). +2. If the type has a **portable** wire format shared with Go, also add the + constant to `sketchlib-go/wire/asapmsgpack/magic_ids.go` with the same + value, and update `SerializeMsgpack` / `DeserializeMsgpack` in the + corresponding Go sketch package. +3. Implement `MessagePackCodec::to_msgpack` / `from_msgpack` (or + `serialize_to_bytes` / `deserialize_from_bytes` for native types) to + prepend / validate the new constant. +4. Add or update round-trip tests. +5. Update this document. diff --git a/src/message_pack_format/portable/countminsketch_topk.rs b/src/message_pack_format/portable/countminsketch_topk.rs index 32d5431..06e3089 100644 --- a/src/message_pack_format/portable/countminsketch_topk.rs +++ b/src/message_pack_format/portable/countminsketch_topk.rs @@ -309,7 +309,7 @@ impl MessagePackCodec for CountMinSketchWithHeap { return Err(MsgPackError::BadMagicId { expected: magic_ids::COUNT_MIN_SKETCH_WITH_HEAP, got: other.copied(), - }) + }); } }; let wire: CountMinSketchWithHeapWire = rmp_serde::from_slice(payload)?; diff --git a/src/message_pack_format/portable/hydra_kll.rs b/src/message_pack_format/portable/hydra_kll.rs index 5e64183..b12ae69 100644 --- a/src/message_pack_format/portable/hydra_kll.rs +++ b/src/message_pack_format/portable/hydra_kll.rs @@ -172,7 +172,7 @@ impl MessagePackCodec for HydraKllSketch { return Err(MsgPackError::BadMagicId { expected: magic_ids::HYDRA_KLL_SKETCH, got: other.copied(), - }) + }); } }; let wire: HydraKllSketchWire = rmp_serde::from_slice(payload)?; diff --git a/src/message_pack_format/portable/kll.rs b/src/message_pack_format/portable/kll.rs index 5b2aeca..045fb61 100644 --- a/src/message_pack_format/portable/kll.rs +++ b/src/message_pack_format/portable/kll.rs @@ -359,7 +359,7 @@ impl MessagePackCodec for KllSketch { return Err(MsgPackError::BadMagicId { expected: magic_ids::KLL_SKETCH, got: other.copied(), - }) + }); } }; let wire: KllSketchData = rmp_serde::from_slice(payload)?; diff --git a/src/message_pack_format/portable/set_aggregator.rs b/src/message_pack_format/portable/set_aggregator.rs index a6131cd..85b2d8a 100644 --- a/src/message_pack_format/portable/set_aggregator.rs +++ b/src/message_pack_format/portable/set_aggregator.rs @@ -93,7 +93,7 @@ impl MessagePackCodec for SetAggregator { return Err(MsgPackError::BadMagicId { expected: magic_ids::SET_AGGREGATOR, got: other.copied(), - }) + }); } }; let wrapper: StringSetOwned = rmp_serde::from_slice(payload)?; diff --git a/src/sketches/countminsketch.rs b/src/sketches/countminsketch.rs index f68e8c3..57bb949 100644 --- a/src/sketches/countminsketch.rs +++ b/src/sketches/countminsketch.rs @@ -281,7 +281,8 @@ impl Deserialize<'de>, Mode, H: SketchHasher> CountM other => Err(RmpDecodeError::Uncategorized(format!( "CountMin magic-ID mismatch: expected 0x{:02x}, got {:?}", crate::message_pack_format::magic_ids::NATIVE_COUNT_MIN, - other.map(|b| format!("0x{b:02x}")) + other + .map(|b| format!("0x{b:02x}")) .unwrap_or_else(|| "empty buffer".to_string()) ))), } diff --git a/src/sketches/countsketch.rs b/src/sketches/countsketch.rs index 6609272..89e255b 100644 --- a/src/sketches/countsketch.rs +++ b/src/sketches/countsketch.rs @@ -304,7 +304,8 @@ where other => Err(RmpDecodeError::Uncategorized(format!( "Count magic-ID mismatch: expected 0x{:02x}, got {:?}", crate::message_pack_format::magic_ids::NATIVE_COUNT_SKETCH, - other.map(|b| format!("0x{b:02x}")) + other + .map(|b| format!("0x{b:02x}")) .unwrap_or_else(|| "empty buffer".to_string()) ))), } diff --git a/src/sketches/countsketch_topk.rs b/src/sketches/countsketch_topk.rs index f19b51f..8ffbc63 100644 --- a/src/sketches/countsketch_topk.rs +++ b/src/sketches/countsketch_topk.rs @@ -1123,7 +1123,8 @@ impl CountL2HH { other => Err(RmpDecodeError::Uncategorized(format!( "CountL2HH magic-ID mismatch: expected 0x{:02x}, got {:?}", crate::message_pack_format::magic_ids::NATIVE_CMS_HEAP, - other.map(|b| format!("0x{b:02x}")) + other + .map(|b| format!("0x{b:02x}")) .unwrap_or_else(|| "empty buffer".to_string()) ))), } diff --git a/src/sketches/ddsketch.rs b/src/sketches/ddsketch.rs index 5669836..1b7c16e 100644 --- a/src/sketches/ddsketch.rs +++ b/src/sketches/ddsketch.rs @@ -192,7 +192,8 @@ impl DDSketch { other => Err(RmpDecodeError::Uncategorized(format!( "DDSketch magic-ID mismatch: expected 0x{:02x}, got {:?}", crate::message_pack_format::magic_ids::NATIVE_DD_SKETCH, - other.map(|b| format!("0x{b:02x}")) + other + .map(|b| format!("0x{b:02x}")) .unwrap_or_else(|| "empty buffer".to_string()) ))), } diff --git a/src/sketches/hll.rs b/src/sketches/hll.rs index 59bafa7..04f315e 100644 --- a/src/sketches/hll.rs +++ b/src/sketches/hll.rs @@ -131,7 +131,8 @@ impl other => Err(RmpDecodeError::Uncategorized(format!( "HyperLogLogImpl magic-ID mismatch: expected 0x{:02x}, got {:?}", crate::message_pack_format::magic_ids::NATIVE_HLL, - other.map(|b| format!("0x{b:02x}")) + other + .map(|b| format!("0x{b:02x}")) .unwrap_or_else(|| "empty buffer".to_string()) ))), } @@ -391,13 +392,12 @@ impl HyperLogLogHIPImpl { /// Deserializes a sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { match bytes.first() { - Some(&crate::message_pack_format::magic_ids::NATIVE_HLL_HIP) => { - from_slice(&bytes[1..]) - } + Some(&crate::message_pack_format::magic_ids::NATIVE_HLL_HIP) => from_slice(&bytes[1..]), other => Err(RmpDecodeError::Uncategorized(format!( "HyperLogLogHIPImpl magic-ID mismatch: expected 0x{:02x}, got {:?}", crate::message_pack_format::magic_ids::NATIVE_HLL_HIP, - other.map(|b| format!("0x{b:02x}")) + other + .map(|b| format!("0x{b:02x}")) .unwrap_or_else(|| "empty buffer".to_string()) ))), } diff --git a/src/sketches/kll.rs b/src/sketches/kll.rs index e060393..0d6caac 100644 --- a/src/sketches/kll.rs +++ b/src/sketches/kll.rs @@ -686,7 +686,8 @@ impl KLL { other => Err(rmp_serde::decode::Error::Uncategorized(format!( "KLL magic-ID mismatch: expected 0x{:02x}, got {:?}", crate::message_pack_format::magic_ids::NATIVE_KLL, - other.map(|b| format!("0x{b:02x}")) + other + .map(|b| format!("0x{b:02x}")) .unwrap_or_else(|| "empty buffer".to_string()) ))), } diff --git a/src/sketches/kll_dynamic.rs b/src/sketches/kll_dynamic.rs index 85a043f..de00ae1 100644 --- a/src/sketches/kll_dynamic.rs +++ b/src/sketches/kll_dynamic.rs @@ -379,7 +379,8 @@ impl KLLDynamic { other => Err(rmp_serde::decode::Error::Uncategorized(format!( "KLLDynamic magic-ID mismatch: expected 0x{:02x}, got {:?}", crate::message_pack_format::magic_ids::NATIVE_KLL_DYNAMIC, - other.map(|b| format!("0x{b:02x}")) + other + .map(|b| format!("0x{b:02x}")) .unwrap_or_else(|| "empty buffer".to_string()) ))), } diff --git a/src/sketches/kmv.rs b/src/sketches/kmv.rs index d31cc27..e0de385 100644 --- a/src/sketches/kmv.rs +++ b/src/sketches/kmv.rs @@ -92,7 +92,8 @@ impl KMV { other => Err(RmpDecodeError::Uncategorized(format!( "KMV magic-ID mismatch: expected 0x{:02x}, got {:?}", crate::message_pack_format::magic_ids::NATIVE_KMV, - other.map(|b| format!("0x{b:02x}")) + other + .map(|b| format!("0x{b:02x}")) .unwrap_or_else(|| "empty buffer".to_string()) ))), } From 2b2c1a1eeff79cd0639f09fa79b2e61b82e8762c Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Wed, 17 Jun 2026 16:25:41 -0400 Subject: [PATCH 04/11] =?UTF-8?q?feat(magic-id):=20cover=20sketch=5Fframew?= =?UTF-8?q?ork=20=E2=80=94=20Hydra=20(0x8a)=20and=20UnivMon=20(0x8b)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the magic-ID rollout across all serialize_to_bytes / deserialize_from_bytes call sites in the codebase. The EH files (eh.rs, eh_sketch_list.rs, eh_univ_optimized.rs) have no serialization methods and need no changes. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- docs/msgpack-magic-ids.md | 2 ++ src/message_pack_format/magic_ids.rs | 6 ++++++ src/sketch_framework/hydra.rs | 20 ++++++++++++++++---- src/sketch_framework/univmon.rs | 20 ++++++++++++++++---- 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/docs/msgpack-magic-ids.md b/docs/msgpack-magic-ids.md index f35fa49..7f61411 100644 --- a/docs/msgpack-magic-ids.md +++ b/docs/msgpack-magic-ids.md @@ -69,6 +69,8 @@ encoding. The two are not interchangeable even for logically equivalent types | `0x87` | `sketches::KLL::serialize_to_bytes` | Compact array format (not named) | | `0x88` | `sketches::KLLDynamic::serialize_to_bytes` | Compact array format (not named) | | `0x89` | `sketches::KMV::serialize_to_bytes` | Named map format (experimental feature) | +| `0x8a` | `sketch_framework::hydra::Hydra::serialize_to_bytes` | Named map format | +| `0x8b` | `sketch_framework::univmon::UnivMon::serialize_to_bytes` | Named map format | ### Relationship between native and portable KLL diff --git a/src/message_pack_format/magic_ids.rs b/src/message_pack_format/magic_ids.rs index 2c1d2ad..97f14fb 100644 --- a/src/message_pack_format/magic_ids.rs +++ b/src/message_pack_format/magic_ids.rs @@ -77,3 +77,9 @@ pub const NATIVE_KLL_DYNAMIC: u8 = 0x88; /// KMV (K-Minimum Values) sketch (`sketches::KMV`). pub const NATIVE_KMV: u8 = 0x89; + +/// Hydra composite sketch (`sketch_framework::hydra`). +pub const NATIVE_HYDRA: u8 = 0x8a; + +/// UnivMon sketch (`sketch_framework::univmon`). +pub const NATIVE_UNIVMON: u8 = 0x8b; diff --git a/src/sketch_framework/hydra.rs b/src/sketch_framework/hydra.rs index 31f61aa..1f305a5 100644 --- a/src/sketch_framework/hydra.rs +++ b/src/sketch_framework/hydra.rs @@ -160,14 +160,26 @@ impl Hydra { self.query_key(key, &HydraQuery::Cdf(threshold)) } - /// Serializes the Hydra sketch (including all counters) into MessagePack bytes. + /// Serializes the Hydra sketch (including all counters) into MessagePack bytes, + /// prefixed with the [`crate::message_pack_format::magic_ids::NATIVE_HYDRA`] magic byte. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - to_vec_named(self) + let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_HYDRA]; + out.extend(to_vec_named(self)?); + Ok(out) } - /// Deserializes a Hydra sketch from MessagePack bytes. + /// Deserializes a Hydra sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - from_slice(bytes) + match bytes.first() { + Some(&crate::message_pack_format::magic_ids::NATIVE_HYDRA) => from_slice(&bytes[1..]), + other => Err(RmpDecodeError::Uncategorized(format!( + "Hydra magic-ID mismatch: expected 0x{:02x}, got {:?}", + crate::message_pack_format::magic_ids::NATIVE_HYDRA, + other + .map(|b| format!("0x{b:02x}")) + .unwrap_or_else(|| "empty buffer".to_string()) + ))), + } } } diff --git a/src/sketch_framework/univmon.rs b/src/sketch_framework/univmon.rs index 9d22f23..7a6e98c 100644 --- a/src/sketch_framework/univmon.rs +++ b/src/sketch_framework/univmon.rs @@ -265,14 +265,26 @@ impl UnivMon { &mut self.hh_layers[layer] } - /// Serializes the UnivMon sketch into MessagePack bytes. + /// Serializes the UnivMon sketch into MessagePack bytes, prefixed with the + /// [`crate::message_pack_format::magic_ids::NATIVE_UNIVMON`] magic byte. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - to_vec_named(self) + let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_UNIVMON]; + out.extend(to_vec_named(self)?); + Ok(out) } - /// Deserializes a UnivMon sketch from MessagePack bytes. + /// Deserializes a UnivMon sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - from_slice(bytes) + match bytes.first() { + Some(&crate::message_pack_format::magic_ids::NATIVE_UNIVMON) => from_slice(&bytes[1..]), + other => Err(RmpDecodeError::Uncategorized(format!( + "UnivMon magic-ID mismatch: expected 0x{:02x}, got {:?}", + crate::message_pack_format::magic_ids::NATIVE_UNIVMON, + other + .map(|b| format!("0x{b:02x}")) + .unwrap_or_else(|| "empty buffer".to_string()) + ))), + } } } From 9961f54dd4002f8207b56e7a03cc60a6f27d72f0 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Wed, 17 Jun 2026 16:42:47 -0400 Subject: [PATCH 05/11] feat(magic-id): Mode/Variant-split impls + hasher discriminant byte MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The native header grows from 1 byte to 2 bytes: [ family+mode byte | hasher byte | ] Key changes: serialize_to_bytes / deserialize_from_bytes are now in Mode-specialized impl blocks instead of a single generic impl, so the first byte encodes both the sketch family and the phantom-type parameter: CountMin<_, RegularPath, _> → 0x81 CountMin<_, FastPath, _> → 0x82 Count<_, RegularPath, _> → 0x83 Count<_, FastPath, _> → 0x84 HyperLogLogImpl → 0x86 HyperLogLogImpl → 0x87 A new hasher_magic_id() method on SketchHasher (default = 0xff = HASHER_UNKNOWN) fills the second byte. DefaultXxHasher returns 0x01 (HASHER_DEFAULT_XX). check_hasher_id(stored) skips the check when either side is 0xff, so custom hashers interoperate without registering. Types without an H parameter (DDSketch, KLL, KLLDynamic, KMV, Hydra, UnivMon, HyperLogLogHIPImpl) always write 0xff or HASHER_DEFAULT_XX as the second byte for a consistent 2-byte header across all native blobs. The native MessagePackCodec shims in message_pack_format/native/ are updated to match the specialized impl split. Docs updated: docs/msgpack-magic-ids.md now shows the 2-byte native header layout and the full updated ID table. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- docs/msgpack-magic-ids.md | 37 ++++-- src/common/hash.rs | 15 +++ src/message_pack_format/magic_ids.rs | 80 +++++++++--- .../native/countminsketch.rs | 17 ++- src/message_pack_format/native/countsketch.rs | 18 ++- src/message_pack_format/native/hll.rs | 22 +++- src/sketch_framework/hydra.rs | 15 ++- src/sketch_framework/univmon.rs | 15 ++- src/sketches/countminsketch.rs | 68 +++++++--- src/sketches/countsketch.rs | 77 +++++++++--- src/sketches/countsketch_topk.rs | 18 +-- src/sketches/ddsketch.rs | 17 +-- src/sketches/hll.rs | 118 +++++++++++++----- src/sketches/kll.rs | 17 +-- src/sketches/kll_dynamic.rs | 17 +-- src/sketches/kmv.rs | 15 ++- 16 files changed, 423 insertions(+), 143 deletions(-) diff --git a/docs/msgpack-magic-ids.md b/docs/msgpack-magic-ids.md index 7f61411..1a9037b 100644 --- a/docs/msgpack-magic-ids.md +++ b/docs/msgpack-magic-ids.md @@ -5,12 +5,24 @@ type-discriminant byte called the **magic ID**. Reading the first byte of any blob is enough to identify what sketch type it contains — no full decode required. +**Portable** (cross-language): ``` ┌─────────────┬───────────────────────────────────────┐ │ magic_id: u8 │ rmp_serde / msgpack payload … │ └─────────────┴───────────────────────────────────────┘ ``` +**Native** (Rust-internal): +``` +┌─────────────┬──────────────┬───────────────────────────────────┐ +│ magic_id: u8 │ hasher_id: u8 │ rmp_serde / msgpack payload … │ +└─────────────┴──────────────┴───────────────────────────────────┘ +``` + +The `hasher_id` byte is `0xFF` (`HASHER_UNKNOWN`) for types without an `H` parameter. +Custom hashers that do not register an ID also store `0xFF` — the mismatch check +is skipped on both sides when either value is `0xFF`. + Magic IDs are **stable** — once assigned, a value is never reused or reassigned. Adding a new sketch type requires a new constant; removing or repurposing an existing constant is a **breaking protocol change**. @@ -60,17 +72,20 @@ encoding. The two are not interchangeable even for logically equivalent types | ID | Rust type / method | Notes | |--------|--------------------------------------------------|-------| -| `0x81` | `sketches::CountMin::serialize_to_bytes` | Named map format; all `Mode` and `H` variants | -| `0x82` | `sketches::Count::serialize_to_bytes` | Named map format; all `Mode` and `H` variants | -| `0x83` | `sketches::CountL2HH::serialize_to_bytes` | Named map format (CMSHeap / heavy-hitter) | -| `0x84` | `sketches::HyperLogLogImpl::serialize_to_bytes` | Named map; Classic and ErtlMLE variants share one ID | -| `0x85` | `sketches::HyperLogLogHIPImpl::serialize_to_bytes` | Named map; HIP-specific accumulators | -| `0x86` | `sketches::DDSketch::serialize_to_bytes` | Named map; distinct from portable `0x05` | -| `0x87` | `sketches::KLL::serialize_to_bytes` | Compact array format (not named) | -| `0x88` | `sketches::KLLDynamic::serialize_to_bytes` | Compact array format (not named) | -| `0x89` | `sketches::KMV::serialize_to_bytes` | Named map format (experimental feature) | -| `0x8a` | `sketch_framework::hydra::Hydra::serialize_to_bytes` | Named map format | -| `0x8b` | `sketch_framework::univmon::UnivMon::serialize_to_bytes` | Named map format | +| `0x81` | `CountMin<_, RegularPath, _>::serialize_to_bytes` | Named map format | +| `0x82` | `CountMin<_, FastPath, _>::serialize_to_bytes` | Named map format | +| `0x83` | `Count<_, RegularPath, _>::serialize_to_bytes` | Named map format | +| `0x84` | `Count<_, FastPath, _>::serialize_to_bytes` | Named map format | +| `0x85` | `CountL2HH::serialize_to_bytes` | Named map format (CMSHeap / heavy-hitter) | +| `0x86` | `HyperLogLogImpl::serialize_to_bytes` | Named map format | +| `0x87` | `HyperLogLogImpl::serialize_to_bytes` | Named map format | +| `0x88` | `HyperLogLogHIPImpl<_>::serialize_to_bytes` | Named map; always `HASHER_DEFAULT_XX` as second byte | +| `0x89` | `sketches::DDSketch::serialize_to_bytes` | Named map; distinct from portable `0x05` | +| `0x8a` | `sketches::KLL::serialize_to_bytes` | Compact array format | +| `0x8b` | `sketches::KLLDynamic::serialize_to_bytes` | Compact array format | +| `0x8c` | `sketches::KMV::serialize_to_bytes` | Named map format (experimental feature) | +| `0x8d` | `sketch_framework::Hydra::serialize_to_bytes` | Named map format | +| `0x8e` | `sketch_framework::UnivMon::serialize_to_bytes` | Named map format | ### Relationship between native and portable KLL diff --git a/src/common/hash.rs b/src/common/hash.rs index ad8cfbc..d2df1ac 100644 --- a/src/common/hash.rs +++ b/src/common/hash.rs @@ -65,6 +65,17 @@ pub trait SketchHasher: Clone + Debug { cols: usize, key: &DataInput, ) -> Self::HashType; + + /// Returns a stable byte discriminant for this hasher, embedded as the + /// second byte of every native serialized header so readers can verify + /// the hash function matches. + /// + /// The default is [`crate::message_pack_format::magic_ids::HASHER_UNKNOWN`] + /// (`0xff`), which disables the mismatch check on both sides. Override + /// this for any hasher that should be verified across process boundaries. + fn hasher_magic_id() -> u8 { + crate::message_pack_format::magic_ids::HASHER_UNKNOWN + } } /// Default hasher using twox_hash (XxHash3). This is the built-in implementation @@ -75,6 +86,10 @@ pub struct DefaultXxHasher; impl SketchHasher for DefaultXxHasher { type HashType = MatrixHashType; + fn hasher_magic_id() -> u8 { + crate::message_pack_format::magic_ids::HASHER_DEFAULT_XX + } + #[inline(always)] fn hash64_seeded(d: usize, key: &DataInput) -> u64 { let seed = SEEDLIST[normalized_seed_idx(d)]; diff --git a/src/message_pack_format/magic_ids.rs b/src/message_pack_format/magic_ids.rs index 97f14fb..d1555b6 100644 --- a/src/message_pack_format/magic_ids.rs +++ b/src/message_pack_format/magic_ids.rs @@ -42,44 +42,88 @@ pub const SET_AGGREGATOR: u8 = 0x08; /// Delta-set aggregator result (added / removed string sets). pub const DELTA_RESULT: u8 = 0x09; +// ── Hasher discriminants ───────────────────────────────────────────────────── +// +// Embedded as the second header byte in every native sketch binary so the +// reader can verify that it uses the same hash function as the writer. +// Custom hashers that do not register an ID are stored as HASHER_UNKNOWN, +// which suppresses the mismatch check on both sides. + +/// Default XxHash3-64 hasher (`DefaultXxHasher`). +pub const HASHER_DEFAULT_XX: u8 = 0x01; + +/// Sentinel for custom hashers with no registered ID — mismatch check is +/// skipped when either the stored value or the expected value is this byte. +pub const HASHER_UNKNOWN: u8 = 0xff; + +/// Validates the stored hasher byte against the expected hasher `H`. +/// +/// The check is skipped (returns `Ok`) when either side is `HASHER_UNKNOWN`, +/// allowing custom hashers to interoperate without requiring a registered ID. +pub(crate) fn check_hasher_id( + stored: u8, +) -> Result<(), rmp_serde::decode::Error> { + let expected = H::hasher_magic_id(); + if stored == HASHER_UNKNOWN || expected == HASHER_UNKNOWN || stored == expected { + return Ok(()); + } + Err(rmp_serde::decode::Error::Uncategorized(format!( + "hasher mismatch: stored 0x{stored:02x}, expected 0x{expected:02x}" + ))) +} + // ── Native (Rust-internal) sketch types ───────────────────────────────────── // // These are the generic sketch types in `crate::sketches`. Their wire format // is produced by `serialize_to_bytes` / `deserialize_from_bytes` and is // internal to Rust — Go (`sketchlib-go`) never reads these bytes directly. // -// They use a separate range (0x81+) to make it visually clear that the byte -// refers to an internal format distinct from the portable cross-language ones. +// The first header byte encodes both the sketch family AND the phantom-type +// parameters (Mode, Variant) that are invisible in the msgpack payload. +// The second header byte encodes the hasher (see HASHER_* above). +// +// Layout: [ family+mode byte | hasher byte | ] +// +// ID range 0x81–0x8f is reserved for native types. + +/// Count-Min sketch with `RegularPath` hashing mode. +pub const NATIVE_COUNT_MIN_REGULAR: u8 = 0x81; + +/// Count-Min sketch with `FastPath` hashing mode. +pub const NATIVE_COUNT_MIN_FAST: u8 = 0x82; + +/// Count Sketch with `RegularPath` hashing mode. +pub const NATIVE_COUNT_SKETCH_REGULAR: u8 = 0x83; -/// Generic Count-Min sketch (`sketches::CountMin<_, _>`). -pub const NATIVE_COUNT_MIN: u8 = 0x81; +/// Count Sketch with `FastPath` hashing mode. +pub const NATIVE_COUNT_SKETCH_FAST: u8 = 0x84; -/// Generic Count Sketch (`sketches::Count<_, _, _>`). -pub const NATIVE_COUNT_SKETCH: u8 = 0x82; +/// Count-Min + heavy-hitter heap (`CountL2HH`). +pub const NATIVE_CMS_HEAP: u8 = 0x85; -/// Count-Min + heavy-hitter heap (`sketches::CMSHeap` / `CountL2HH`). -pub const NATIVE_CMS_HEAP: u8 = 0x83; +/// HyperLogLog Classic (HLL++) estimator (`HyperLogLogImpl`). +pub const NATIVE_HLL_CLASSIC: u8 = 0x86; -/// Generic HyperLogLog (`sketches::HyperLogLogImpl<_, _, _>` — Classic and ErtlMLE variants). -pub const NATIVE_HLL: u8 = 0x84; +/// HyperLogLog ErtlMLE estimator (`HyperLogLogImpl`). +pub const NATIVE_HLL_ERTL_MLE: u8 = 0x87; -/// HyperLogLog HIP variant (`sketches::HyperLogLogHIPImpl<_>`). -pub const NATIVE_HLL_HIP: u8 = 0x85; +/// HyperLogLog HIP variant (`HyperLogLogHIPImpl<_>`). +pub const NATIVE_HLL_HIP: u8 = 0x88; /// DDSketch (`sketches::DDSketch`). -pub const NATIVE_DD_SKETCH: u8 = 0x86; +pub const NATIVE_DD_SKETCH: u8 = 0x89; /// KLL quantile sketch (`sketches::kll::KLL`). -pub const NATIVE_KLL: u8 = 0x87; +pub const NATIVE_KLL: u8 = 0x8a; /// Dynamic KLL quantile sketch (`sketches::kll_dynamic::KLLDynamic`). -pub const NATIVE_KLL_DYNAMIC: u8 = 0x88; +pub const NATIVE_KLL_DYNAMIC: u8 = 0x8b; /// KMV (K-Minimum Values) sketch (`sketches::KMV`). -pub const NATIVE_KMV: u8 = 0x89; +pub const NATIVE_KMV: u8 = 0x8c; /// Hydra composite sketch (`sketch_framework::hydra`). -pub const NATIVE_HYDRA: u8 = 0x8a; +pub const NATIVE_HYDRA: u8 = 0x8d; /// UnivMon sketch (`sketch_framework::univmon`). -pub const NATIVE_UNIVMON: u8 = 0x8b; +pub const NATIVE_UNIVMON: u8 = 0x8e; diff --git a/src/message_pack_format/native/countminsketch.rs b/src/message_pack_format/native/countminsketch.rs index bed39e2..4471d24 100644 --- a/src/message_pack_format/native/countminsketch.rs +++ b/src/message_pack_format/native/countminsketch.rs @@ -4,9 +4,22 @@ use serde::{Deserialize, Serialize}; use crate::message_pack_format::{Error, MessagePackCodec}; use crate::sketches::countminsketch::CountMin; -use crate::{MatrixStorage, SketchHasher}; +use crate::{FastPath, MatrixStorage, RegularPath, SketchHasher}; -impl MessagePackCodec for CountMin +impl MessagePackCodec for CountMin +where + S: MatrixStorage + Serialize + for<'de> Deserialize<'de>, +{ + fn to_msgpack(&self) -> Result, Error> { + Ok(self.serialize_to_bytes()?) + } + + fn from_msgpack(bytes: &[u8]) -> Result { + Ok(Self::deserialize_from_bytes(bytes)?) + } +} + +impl MessagePackCodec for CountMin where S: MatrixStorage + Serialize + for<'de> Deserialize<'de>, { diff --git a/src/message_pack_format/native/countsketch.rs b/src/message_pack_format/native/countsketch.rs index 03a2662..453d106 100644 --- a/src/message_pack_format/native/countsketch.rs +++ b/src/message_pack_format/native/countsketch.rs @@ -4,9 +4,23 @@ use serde::{Deserialize, Serialize}; use crate::message_pack_format::{Error, MessagePackCodec}; use crate::sketches::countsketch::{Count, CountSketchCounter}; -use crate::{MatrixStorage, SketchHasher}; +use crate::{FastPath, MatrixStorage, RegularPath, SketchHasher}; -impl MessagePackCodec for Count +impl MessagePackCodec for Count +where + S: MatrixStorage + Serialize + for<'de> Deserialize<'de>, + C: CountSketchCounter, +{ + fn to_msgpack(&self) -> Result, Error> { + Ok(self.serialize_to_bytes()?) + } + + fn from_msgpack(bytes: &[u8]) -> Result { + Ok(Self::deserialize_from_bytes(bytes)?) + } +} + +impl MessagePackCodec for Count where S: MatrixStorage + Serialize + for<'de> Deserialize<'de>, C: CountSketchCounter, diff --git a/src/message_pack_format/native/hll.rs b/src/message_pack_format/native/hll.rs index 36f76e9..5c624f7 100644 --- a/src/message_pack_format/native/hll.rs +++ b/src/message_pack_format/native/hll.rs @@ -1,11 +1,27 @@ //! Native MessagePack codec impls for [`crate::sketches::hll`]. +use serde::{Deserialize, Serialize}; + use crate::message_pack_format::{Error, MessagePackCodec}; -use crate::sketches::hll::{HyperLogLogHIPImpl, HyperLogLogImpl}; +use crate::sketches::hll::{Classic, ErtlMLE, HyperLogLogHIPImpl, HyperLogLogImpl}; use crate::{HllRegisterStorage, SketchHasher}; -impl MessagePackCodec - for HyperLogLogImpl +impl MessagePackCodec for HyperLogLogImpl +where + Registers: HllRegisterStorage + Serialize + for<'de> Deserialize<'de>, +{ + fn to_msgpack(&self) -> Result, Error> { + Ok(self.serialize_to_bytes()?) + } + + fn from_msgpack(bytes: &[u8]) -> Result { + Ok(Self::deserialize_from_bytes(bytes)?) + } +} + +impl MessagePackCodec for HyperLogLogImpl +where + Registers: HllRegisterStorage + Serialize + for<'de> Deserialize<'de>, { fn to_msgpack(&self) -> Result, Error> { Ok(self.serialize_to_bytes()?) diff --git a/src/sketch_framework/hydra.rs b/src/sketch_framework/hydra.rs index 1f305a5..bcd0ecc 100644 --- a/src/sketch_framework/hydra.rs +++ b/src/sketch_framework/hydra.rs @@ -163,19 +163,22 @@ impl Hydra { /// Serializes the Hydra sketch (including all counters) into MessagePack bytes, /// prefixed with the [`crate::message_pack_format::magic_ids::NATIVE_HYDRA`] magic byte. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_HYDRA]; + use crate::message_pack_format::magic_ids; + let mut out = vec![magic_ids::NATIVE_HYDRA, magic_ids::HASHER_UNKNOWN]; out.extend(to_vec_named(self)?); Ok(out) } /// Deserializes a Hydra sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - match bytes.first() { - Some(&crate::message_pack_format::magic_ids::NATIVE_HYDRA) => from_slice(&bytes[1..]), - other => Err(RmpDecodeError::Uncategorized(format!( + use crate::message_pack_format::magic_ids; + match bytes { + [id, _hasher, rest @ ..] if *id == magic_ids::NATIVE_HYDRA => from_slice(rest), + _ => Err(RmpDecodeError::Uncategorized(format!( "Hydra magic-ID mismatch: expected 0x{:02x}, got {:?}", - crate::message_pack_format::magic_ids::NATIVE_HYDRA, - other + magic_ids::NATIVE_HYDRA, + bytes + .first() .map(|b| format!("0x{b:02x}")) .unwrap_or_else(|| "empty buffer".to_string()) ))), diff --git a/src/sketch_framework/univmon.rs b/src/sketch_framework/univmon.rs index 7a6e98c..8406ceb 100644 --- a/src/sketch_framework/univmon.rs +++ b/src/sketch_framework/univmon.rs @@ -268,19 +268,22 @@ impl UnivMon { /// Serializes the UnivMon sketch into MessagePack bytes, prefixed with the /// [`crate::message_pack_format::magic_ids::NATIVE_UNIVMON`] magic byte. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_UNIVMON]; + use crate::message_pack_format::magic_ids; + let mut out = vec![magic_ids::NATIVE_UNIVMON, magic_ids::HASHER_UNKNOWN]; out.extend(to_vec_named(self)?); Ok(out) } /// Deserializes a UnivMon sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - match bytes.first() { - Some(&crate::message_pack_format::magic_ids::NATIVE_UNIVMON) => from_slice(&bytes[1..]), - other => Err(RmpDecodeError::Uncategorized(format!( + use crate::message_pack_format::magic_ids; + match bytes { + [id, _hasher, rest @ ..] if *id == magic_ids::NATIVE_UNIVMON => from_slice(rest), + _ => Err(RmpDecodeError::Uncategorized(format!( "UnivMon magic-ID mismatch: expected 0x{:02x}, got {:?}", - crate::message_pack_format::magic_ids::NATIVE_UNIVMON, - other + magic_ids::NATIVE_UNIVMON, + bytes + .first() .map(|b| format!("0x{b:02x}")) .unwrap_or_else(|| "empty buffer".to_string()) ))), diff --git a/src/sketches/countminsketch.rs b/src/sketches/countminsketch.rs index 57bb949..094064e 100644 --- a/src/sketches/countminsketch.rs +++ b/src/sketches/countminsketch.rs @@ -260,28 +260,68 @@ impl CountMin { } } -// Serialization helpers for CountMin. -impl CountMin { - /// Serializes the sketch into MessagePack bytes, prefixed with the - /// [`crate::message_pack_format::magic_ids::NATIVE_COUNT_MIN`] magic byte. +// Serialization helpers for CountMin — one impl per Mode so the magic byte +// encodes both the sketch family and the hashing strategy. +// +// Wire layout: [ mode_id: u8 | hasher_id: u8 | ] + +impl CountMin { + /// Serializes the sketch into MessagePack bytes. + /// Header: `[NATIVE_COUNT_MIN_REGULAR, hasher_id]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_COUNT_MIN]; + use crate::message_pack_format::magic_ids; + let mut out = vec![magic_ids::NATIVE_COUNT_MIN_REGULAR, H::hasher_magic_id()]; out.extend(to_vec_named(self)?); Ok(out) } } -impl Deserialize<'de>, Mode, H: SketchHasher> CountMin { - /// Deserializes a sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. +impl Deserialize<'de>, H: SketchHasher> CountMin { + /// Deserializes a sketch from bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - match bytes.first() { - Some(&crate::message_pack_format::magic_ids::NATIVE_COUNT_MIN) => { - from_slice(&bytes[1..]) + use crate::message_pack_format::magic_ids; + match bytes { + [id, hasher, rest @ ..] if *id == magic_ids::NATIVE_COUNT_MIN_REGULAR => { + magic_ids::check_hasher_id::(*hasher)?; + from_slice(rest) } - other => Err(RmpDecodeError::Uncategorized(format!( - "CountMin magic-ID mismatch: expected 0x{:02x}, got {:?}", - crate::message_pack_format::magic_ids::NATIVE_COUNT_MIN, - other + _ => Err(RmpDecodeError::Uncategorized(format!( + "CountMin magic-ID mismatch: expected 0x{:02x}, got {:?}", + magic_ids::NATIVE_COUNT_MIN_REGULAR, + bytes + .first() + .map(|b| format!("0x{b:02x}")) + .unwrap_or_else(|| "empty buffer".to_string()) + ))), + } + } +} + +impl CountMin { + /// Serializes the sketch into MessagePack bytes. + /// Header: `[NATIVE_COUNT_MIN_FAST, hasher_id]`. + pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { + use crate::message_pack_format::magic_ids; + let mut out = vec![magic_ids::NATIVE_COUNT_MIN_FAST, H::hasher_magic_id()]; + out.extend(to_vec_named(self)?); + Ok(out) + } +} + +impl Deserialize<'de>, H: SketchHasher> CountMin { + /// Deserializes a sketch from bytes produced by [`Self::serialize_to_bytes`]. + pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { + use crate::message_pack_format::magic_ids; + match bytes { + [id, hasher, rest @ ..] if *id == magic_ids::NATIVE_COUNT_MIN_FAST => { + magic_ids::check_hasher_id::(*hasher)?; + from_slice(rest) + } + _ => Err(RmpDecodeError::Uncategorized(format!( + "CountMin magic-ID mismatch: expected 0x{:02x}, got {:?}", + magic_ids::NATIVE_COUNT_MIN_FAST, + bytes + .first() .map(|b| format!("0x{b:02x}")) .unwrap_or_else(|| "empty buffer".to_string()) ))), diff --git a/src/sketches/countsketch.rs b/src/sketches/countsketch.rs index 89e255b..26aa052 100644 --- a/src/sketches/countsketch.rs +++ b/src/sketches/countsketch.rs @@ -274,37 +274,84 @@ where } } -// Serialization helpers for Count. -impl Count +// Serialization helpers for Count — one impl per Mode so the magic byte +// encodes both the sketch family and the hashing strategy. +// +// Wire layout: [ mode_id: u8 | hasher_id: u8 | ] + +impl Count where S: MatrixStorage + Serialize, C: CountSketchCounter, { - /// Serializes the sketch into MessagePack bytes, prefixed with the - /// [`crate::message_pack_format::magic_ids::NATIVE_COUNT_SKETCH`] magic byte. + /// Serializes the sketch into MessagePack bytes. + /// Header: `[NATIVE_COUNT_SKETCH_REGULAR, hasher_id]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_COUNT_SKETCH]; + use crate::message_pack_format::magic_ids; + let mut out = vec![magic_ids::NATIVE_COUNT_SKETCH_REGULAR, H::hasher_magic_id()]; out.extend(to_vec_named(self)?); Ok(out) } } -// Deserialization helpers for Count. -impl Count +impl Count +where + S: MatrixStorage + for<'de> Deserialize<'de>, + C: CountSketchCounter, +{ + /// Deserializes a sketch from bytes produced by [`Self::serialize_to_bytes`]. + pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { + use crate::message_pack_format::magic_ids; + match bytes { + [id, hasher, rest @ ..] if *id == magic_ids::NATIVE_COUNT_SKETCH_REGULAR => { + magic_ids::check_hasher_id::(*hasher)?; + from_slice(rest) + } + _ => Err(RmpDecodeError::Uncategorized(format!( + "Count magic-ID mismatch: expected 0x{:02x}, got {:?}", + magic_ids::NATIVE_COUNT_SKETCH_REGULAR, + bytes + .first() + .map(|b| format!("0x{b:02x}")) + .unwrap_or_else(|| "empty buffer".to_string()) + ))), + } + } +} + +impl Count +where + S: MatrixStorage + Serialize, + C: CountSketchCounter, +{ + /// Serializes the sketch into MessagePack bytes. + /// Header: `[NATIVE_COUNT_SKETCH_FAST, hasher_id]`. + pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { + use crate::message_pack_format::magic_ids; + let mut out = vec![magic_ids::NATIVE_COUNT_SKETCH_FAST, H::hasher_magic_id()]; + out.extend(to_vec_named(self)?); + Ok(out) + } +} + +impl Count where S: MatrixStorage + for<'de> Deserialize<'de>, C: CountSketchCounter, { - /// Deserializes a sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. + /// Deserializes a sketch from bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - match bytes.first() { - Some(&crate::message_pack_format::magic_ids::NATIVE_COUNT_SKETCH) => { - from_slice(&bytes[1..]) + use crate::message_pack_format::magic_ids; + match bytes { + [id, hasher, rest @ ..] if *id == magic_ids::NATIVE_COUNT_SKETCH_FAST => { + magic_ids::check_hasher_id::(*hasher)?; + from_slice(rest) } - other => Err(RmpDecodeError::Uncategorized(format!( - "Count magic-ID mismatch: expected 0x{:02x}, got {:?}", - crate::message_pack_format::magic_ids::NATIVE_COUNT_SKETCH, - other + _ => Err(RmpDecodeError::Uncategorized(format!( + "Count magic-ID mismatch: expected 0x{:02x}, got {:?}", + magic_ids::NATIVE_COUNT_SKETCH_FAST, + bytes + .first() .map(|b| format!("0x{b:02x}")) .unwrap_or_else(|| "empty buffer".to_string()) ))), diff --git a/src/sketches/countsketch_topk.rs b/src/sketches/countsketch_topk.rs index 8ffbc63..fe14bf5 100644 --- a/src/sketches/countsketch_topk.rs +++ b/src/sketches/countsketch_topk.rs @@ -1109,21 +1109,25 @@ impl CountL2HH { /// Serializes the CountL2HH sketch into MessagePack bytes, prefixed with the /// [`crate::message_pack_format::magic_ids::NATIVE_CMS_HEAP`] magic byte. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_CMS_HEAP]; + use crate::message_pack_format::magic_ids; + let mut out = vec![magic_ids::NATIVE_CMS_HEAP, H::hasher_magic_id()]; out.extend(to_vec_named(self)?); Ok(out) } /// Deserializes a CountL2HH sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - match bytes.first() { - Some(&crate::message_pack_format::magic_ids::NATIVE_CMS_HEAP) => { - from_slice(&bytes[1..]) + use crate::message_pack_format::magic_ids; + match bytes { + [id, hasher, rest @ ..] if *id == magic_ids::NATIVE_CMS_HEAP => { + magic_ids::check_hasher_id::(*hasher)?; + from_slice(rest) } - other => Err(RmpDecodeError::Uncategorized(format!( + _ => Err(RmpDecodeError::Uncategorized(format!( "CountL2HH magic-ID mismatch: expected 0x{:02x}, got {:?}", - crate::message_pack_format::magic_ids::NATIVE_CMS_HEAP, - other + magic_ids::NATIVE_CMS_HEAP, + bytes + .first() .map(|b| format!("0x{b:02x}")) .unwrap_or_else(|| "empty buffer".to_string()) ))), diff --git a/src/sketches/ddsketch.rs b/src/sketches/ddsketch.rs index 1b7c16e..21c392f 100644 --- a/src/sketches/ddsketch.rs +++ b/src/sketches/ddsketch.rs @@ -168,7 +168,8 @@ impl DDSketch { /// Serializes the sketch to a MessagePack byte vector, prefixed with the /// [`crate::message_pack_format::magic_ids::NATIVE_DD_SKETCH`] magic byte. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_DD_SKETCH]; + use crate::message_pack_format::magic_ids; + let mut out = vec![magic_ids::NATIVE_DD_SKETCH, magic_ids::HASHER_UNKNOWN]; out.extend(to_vec_named(self)?); Ok(out) } @@ -183,16 +184,18 @@ impl DDSketch { /// reconstructed from the per-bucket representative values, accurate /// to within the sketch's α relative-accuracy bound. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - match bytes.first() { - Some(&crate::message_pack_format::magic_ids::NATIVE_DD_SKETCH) => { - let mut sk: Self = from_slice(&bytes[1..])?; + use crate::message_pack_format::magic_ids; + match bytes { + [id, _hasher, rest @ ..] if *id == magic_ids::NATIVE_DD_SKETCH => { + let mut sk: Self = from_slice(rest)?; sk.recompute_scalars_from_store(); Ok(sk) } - other => Err(RmpDecodeError::Uncategorized(format!( + _ => Err(RmpDecodeError::Uncategorized(format!( "DDSketch magic-ID mismatch: expected 0x{:02x}, got {:?}", - crate::message_pack_format::magic_ids::NATIVE_DD_SKETCH, - other + magic_ids::NATIVE_DD_SKETCH, + bytes + .first() .map(|b| format!("0x{b:02x}")) .unwrap_or_else(|| "empty buffer".to_string()) ))), diff --git a/src/sketches/hll.rs b/src/sketches/hll.rs index 04f315e..25b6bbd 100644 --- a/src/sketches/hll.rs +++ b/src/sketches/hll.rs @@ -116,28 +116,6 @@ impl } } - /// Serializes the sketch into MessagePack bytes, prefixed with the - /// [`crate::message_pack_format::magic_ids::NATIVE_HLL`] magic byte. - pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_HLL]; - out.extend(to_vec_named(self)?); - Ok(out) - } - - /// Deserializes a sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. - pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - match bytes.first() { - Some(&crate::message_pack_format::magic_ids::NATIVE_HLL) => from_slice(&bytes[1..]), - other => Err(RmpDecodeError::Uncategorized(format!( - "HyperLogLogImpl magic-ID mismatch: expected 0x{:02x}, got {:?}", - crate::message_pack_format::magic_ids::NATIVE_HLL, - other - .map(|b| format!("0x{b:02x}")) - .unwrap_or_else(|| "empty buffer".to_string()) - ))), - } - } - /// Borrow the raw register byte slice (one byte per register). pub fn registers_as_slice(&self) -> &[u8] { self.registers.as_slice() @@ -182,6 +160,81 @@ impl } } +// Serialization — split by Variant so the magic byte encodes the estimator. +// Wire layout: [ variant_id: u8 | hasher_id: u8 | ] + +impl + HyperLogLogImpl +{ + /// Serializes the sketch into MessagePack bytes. + /// Header: `[NATIVE_HLL_CLASSIC, hasher_id]`. + pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { + use crate::message_pack_format::magic_ids; + let mut out = vec![magic_ids::NATIVE_HLL_CLASSIC, H::hasher_magic_id()]; + out.extend(to_vec_named(self)?); + Ok(out) + } +} + +impl Deserialize<'de>, H: SketchHasher> + HyperLogLogImpl +{ + /// Deserializes a sketch from bytes produced by [`Self::serialize_to_bytes`]. + pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { + use crate::message_pack_format::magic_ids; + match bytes { + [id, hasher, rest @ ..] if *id == magic_ids::NATIVE_HLL_CLASSIC => { + magic_ids::check_hasher_id::(*hasher)?; + from_slice(rest) + } + _ => Err(RmpDecodeError::Uncategorized(format!( + "HyperLogLogImpl magic-ID mismatch: expected 0x{:02x}, got {:?}", + magic_ids::NATIVE_HLL_CLASSIC, + bytes + .first() + .map(|b| format!("0x{b:02x}")) + .unwrap_or_else(|| "empty buffer".to_string()) + ))), + } + } +} + +impl + HyperLogLogImpl +{ + /// Serializes the sketch into MessagePack bytes. + /// Header: `[NATIVE_HLL_ERTL_MLE, hasher_id]`. + pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { + use crate::message_pack_format::magic_ids; + let mut out = vec![magic_ids::NATIVE_HLL_ERTL_MLE, H::hasher_magic_id()]; + out.extend(to_vec_named(self)?); + Ok(out) + } +} + +impl Deserialize<'de>, H: SketchHasher> + HyperLogLogImpl +{ + /// Deserializes a sketch from bytes produced by [`Self::serialize_to_bytes`]. + pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { + use crate::message_pack_format::magic_ids; + match bytes { + [id, hasher, rest @ ..] if *id == magic_ids::NATIVE_HLL_ERTL_MLE => { + magic_ids::check_hasher_id::(*hasher)?; + from_slice(rest) + } + _ => Err(RmpDecodeError::Uncategorized(format!( + "HyperLogLogImpl magic-ID mismatch: expected 0x{:02x}, got {:?}", + magic_ids::NATIVE_HLL_ERTL_MLE, + bytes + .first() + .map(|b| format!("0x{b:02x}")) + .unwrap_or_else(|| "empty buffer".to_string()) + ))), + } + } +} + // DataInput adapters (hashing + batch helpers). impl HyperLogLogImpl @@ -381,22 +434,25 @@ impl HyperLogLogHIPImpl { self.est as usize } - /// Serializes the sketch into MessagePack bytes, prefixed with the - /// [`crate::message_pack_format::magic_ids::NATIVE_HLL_HIP`] magic byte. + /// Serializes the sketch into MessagePack bytes. + /// Header: `[NATIVE_HLL_HIP, HASHER_DEFAULT_XX]` — HIP always uses DefaultXxHasher. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_HLL_HIP]; + use crate::message_pack_format::magic_ids; + let mut out = vec![magic_ids::NATIVE_HLL_HIP, magic_ids::HASHER_DEFAULT_XX]; out.extend(to_vec_named(self)?); Ok(out) } - /// Deserializes a sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. + /// Deserializes a sketch from bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - match bytes.first() { - Some(&crate::message_pack_format::magic_ids::NATIVE_HLL_HIP) => from_slice(&bytes[1..]), - other => Err(RmpDecodeError::Uncategorized(format!( + use crate::message_pack_format::magic_ids; + match bytes { + [id, _hasher, rest @ ..] if *id == magic_ids::NATIVE_HLL_HIP => from_slice(rest), + _ => Err(RmpDecodeError::Uncategorized(format!( "HyperLogLogHIPImpl magic-ID mismatch: expected 0x{:02x}, got {:?}", - crate::message_pack_format::magic_ids::NATIVE_HLL_HIP, - other + magic_ids::NATIVE_HLL_HIP, + bytes + .first() .map(|b| format!("0x{b:02x}")) .unwrap_or_else(|| "empty buffer".to_string()) ))), diff --git a/src/sketches/kll.rs b/src/sketches/kll.rs index 0d6caac..5b6c20f 100644 --- a/src/sketches/kll.rs +++ b/src/sketches/kll.rs @@ -668,7 +668,8 @@ impl KLL { where T: Serialize, { - let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_KLL]; + use crate::message_pack_format::magic_ids; + let mut out = vec![magic_ids::NATIVE_KLL, magic_ids::HASHER_UNKNOWN]; out.extend(rmp_serde::to_vec(self)?); Ok(out) } @@ -679,14 +680,14 @@ impl KLL { where T: for<'de> Deserialize<'de>, { - match bytes.first() { - Some(&crate::message_pack_format::magic_ids::NATIVE_KLL) => { - rmp_serde::from_slice(&bytes[1..]) - } - other => Err(rmp_serde::decode::Error::Uncategorized(format!( + use crate::message_pack_format::magic_ids; + match bytes { + [id, _hasher, rest @ ..] if *id == magic_ids::NATIVE_KLL => rmp_serde::from_slice(rest), + _ => Err(rmp_serde::decode::Error::Uncategorized(format!( "KLL magic-ID mismatch: expected 0x{:02x}, got {:?}", - crate::message_pack_format::magic_ids::NATIVE_KLL, - other + magic_ids::NATIVE_KLL, + bytes + .first() .map(|b| format!("0x{b:02x}")) .unwrap_or_else(|| "empty buffer".to_string()) ))), diff --git a/src/sketches/kll_dynamic.rs b/src/sketches/kll_dynamic.rs index de00ae1..ca1a739 100644 --- a/src/sketches/kll_dynamic.rs +++ b/src/sketches/kll_dynamic.rs @@ -359,7 +359,8 @@ impl KLLDynamic { where T: Serialize, { - let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_KLL_DYNAMIC]; + use crate::message_pack_format::magic_ids; + let mut out = vec![magic_ids::NATIVE_KLL_DYNAMIC, magic_ids::HASHER_UNKNOWN]; out.extend(rmp_serde::to_vec(self)?); Ok(out) } @@ -369,17 +370,19 @@ impl KLLDynamic { where T: for<'de> Deserialize<'de>, { - match bytes.first() { - Some(&crate::message_pack_format::magic_ids::NATIVE_KLL_DYNAMIC) => { - rmp_serde::from_slice(&bytes[1..]).map(|mut sketch: KLLDynamic| { + use crate::message_pack_format::magic_ids; + match bytes { + [id, _hasher, rest @ ..] if *id == magic_ids::NATIVE_KLL_DYNAMIC => { + rmp_serde::from_slice(rest).map(|mut sketch: KLLDynamic| { sketch.rebuild_capacity_cache(); sketch }) } - other => Err(rmp_serde::decode::Error::Uncategorized(format!( + _ => Err(rmp_serde::decode::Error::Uncategorized(format!( "KLLDynamic magic-ID mismatch: expected 0x{:02x}, got {:?}", - crate::message_pack_format::magic_ids::NATIVE_KLL_DYNAMIC, - other + magic_ids::NATIVE_KLL_DYNAMIC, + bytes + .first() .map(|b| format!("0x{b:02x}")) .unwrap_or_else(|| "empty buffer".to_string()) ))), diff --git a/src/sketches/kmv.rs b/src/sketches/kmv.rs index e0de385..6f5141f 100644 --- a/src/sketches/kmv.rs +++ b/src/sketches/kmv.rs @@ -80,19 +80,22 @@ impl KMV { /// Serializes the sketch into MessagePack bytes, prefixed with the /// [`crate::message_pack_format::magic_ids::NATIVE_KMV`] magic byte. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - let mut out = vec![crate::message_pack_format::magic_ids::NATIVE_KMV]; + use crate::message_pack_format::magic_ids; + let mut out = vec![magic_ids::NATIVE_KMV, magic_ids::HASHER_UNKNOWN]; out.extend(to_vec_named(self)?); Ok(out) } /// Deserializes a sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - match bytes.first() { - Some(&crate::message_pack_format::magic_ids::NATIVE_KMV) => from_slice(&bytes[1..]), - other => Err(RmpDecodeError::Uncategorized(format!( + use crate::message_pack_format::magic_ids; + match bytes { + [id, _hasher, rest @ ..] if *id == magic_ids::NATIVE_KMV => from_slice(rest), + _ => Err(RmpDecodeError::Uncategorized(format!( "KMV magic-ID mismatch: expected 0x{:02x}, got {:?}", - crate::message_pack_format::magic_ids::NATIVE_KMV, - other + magic_ids::NATIVE_KMV, + bytes + .first() .map(|b| format!("0x{b:02x}")) .unwrap_or_else(|| "empty buffer".to_string()) ))), From f2b26b38eee8a37e4fcbfd446e861e4f76ba3aba Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Wed, 17 Jun 2026 16:55:12 -0400 Subject: [PATCH 06/11] test: add FastPath round-trips and Mode/Variant mismatch rejection tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the coverage gap: the earlier split-impl work only had RegularPath round-trip tests. New tests: count_min_fast_path_round_trip_serialization — verifies magic bytes 0x82 / HASHER_DEFAULT_XX and data survives serialize→deserialize count_min_mode_mismatch_is_rejected — FastPath bytes → RegularPath decoder must error count_sketch_fast_path_round_trip_serialization — same for Count count_sketch_mode_mismatch_is_rejected hll_magic_bytes_are_variant_specific — Classic=0x86, ErtlMLE=0x87 hll_variant_mismatch_is_rejected — ErtlMLE bytes → Classic decoder must error Co-Authored-By: Claude Sonnet 4.6 (1M context) --- src/sketches/countminsketch.rs | 43 ++++++++++++++++++++++++++++++++++ src/sketches/countsketch.rs | 42 +++++++++++++++++++++++++++++++++ src/sketches/hll.rs | 30 ++++++++++++++++++++++++ 3 files changed, 115 insertions(+) diff --git a/src/sketches/countminsketch.rs b/src/sketches/countminsketch.rs index 094064e..49c32a8 100644 --- a/src/sketches/countminsketch.rs +++ b/src/sketches/countminsketch.rs @@ -1013,4 +1013,47 @@ mod tests { decoded.as_storage().as_slice() ); } + + #[test] + fn count_min_fast_path_round_trip_serialization() { + use crate::message_pack_format::magic_ids; + + let mut sketch = CountMin::, FastPath>::with_dimensions(3, 8); + sketch.insert(&DataInput::U64(42)); + sketch.insert(&DataInput::U64(7)); + + let encoded = sketch + .serialize_to_bytes() + .expect("serialize CountMin FastPath"); + assert_eq!( + encoded[0], + magic_ids::NATIVE_COUNT_MIN_FAST, + "wrong magic id" + ); + assert_eq!(encoded[1], magic_ids::HASHER_DEFAULT_XX, "wrong hasher id"); + + let decoded = CountMin::, FastPath>::deserialize_from_bytes(&encoded) + .expect("deserialize CountMin FastPath"); + + assert_eq!(sketch.rows(), decoded.rows()); + assert_eq!(sketch.cols(), decoded.cols()); + assert_eq!( + sketch.as_storage().as_slice(), + decoded.as_storage().as_slice() + ); + } + + #[test] + fn count_min_mode_mismatch_is_rejected() { + let mut sketch = CountMin::, FastPath>::with_dimensions(3, 8); + sketch.insert(&DataInput::U64(1)); + + let fast_bytes = sketch.serialize_to_bytes().unwrap(); + // Feeding FastPath bytes to a RegularPath decoder must error. + let result = CountMin::, RegularPath>::deserialize_from_bytes(&fast_bytes); + assert!( + result.is_err(), + "expected error decoding FastPath bytes as RegularPath" + ); + } } diff --git a/src/sketches/countsketch.rs b/src/sketches/countsketch.rs index 26aa052..f50840e 100644 --- a/src/sketches/countsketch.rs +++ b/src/sketches/countsketch.rs @@ -1161,4 +1161,46 @@ mod tests { decoded.as_storage().as_slice() ); } + + #[test] + fn count_sketch_fast_path_round_trip_serialization() { + use crate::message_pack_format::magic_ids; + + let mut sketch = Count::, FastPath>::with_dimensions(3, 8); + sketch.insert(&DataInput::U64(42)); + sketch.insert(&DataInput::U64(7)); + + let encoded = sketch + .serialize_to_bytes() + .expect("serialize Count FastPath"); + assert_eq!( + encoded[0], + magic_ids::NATIVE_COUNT_SKETCH_FAST, + "wrong magic id" + ); + assert_eq!(encoded[1], magic_ids::HASHER_DEFAULT_XX, "wrong hasher id"); + + let decoded = Count::, FastPath>::deserialize_from_bytes(&encoded) + .expect("deserialize Count FastPath"); + + assert_eq!(sketch.rows(), decoded.rows()); + assert_eq!(sketch.cols(), decoded.cols()); + assert_eq!( + sketch.as_storage().as_slice(), + decoded.as_storage().as_slice() + ); + } + + #[test] + fn count_sketch_mode_mismatch_is_rejected() { + let mut sketch = Count::, FastPath>::with_dimensions(3, 8); + sketch.insert(&DataInput::U64(1)); + + let fast_bytes = sketch.serialize_to_bytes().unwrap(); + let result = Count::, RegularPath>::deserialize_from_bytes(&fast_bytes); + assert!( + result.is_err(), + "expected error decoding FastPath bytes as RegularPath" + ); + } } diff --git a/src/sketches/hll.rs b/src/sketches/hll.rs index 25b6bbd..d2ac7ca 100644 --- a/src/sketches/hll.rs +++ b/src/sketches/hll.rs @@ -755,6 +755,36 @@ mod tests { assert_serialization_round_trip::("HllDsP12"); } + #[test] + fn hll_magic_bytes_are_variant_specific() { + use crate::message_pack_format::magic_ids; + + let classic_bytes = HyperLogLog::::default() + .serialize_to_bytes() + .unwrap(); + let ertl_bytes = HyperLogLog::::default() + .serialize_to_bytes() + .unwrap(); + + assert_eq!(classic_bytes[0], magic_ids::NATIVE_HLL_CLASSIC); + assert_eq!(ertl_bytes[0], magic_ids::NATIVE_HLL_ERTL_MLE); + // Sanity: they are different. + assert_ne!(classic_bytes[0], ertl_bytes[0]); + } + + #[test] + fn hll_variant_mismatch_is_rejected() { + let ertl_bytes = HyperLogLog::::default() + .serialize_to_bytes() + .unwrap(); + // Feeding ErtlMLE bytes to a Classic decoder must error. + let result = HyperLogLog::::deserialize_from_bytes(&ertl_bytes); + assert!( + result.is_err(), + "expected error decoding ErtlMLE bytes as Classic" + ); + } + // insert 10 values and check corresponding counter is updated #[test] fn hll_correctness_test() { From add393551ce0b6485b34f274e5a0940af4d60cb1 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Sun, 21 Jun 2026 02:54:39 -0400 Subject: [PATCH 07/11] feat(msgpack): replace bare magic bytes with ASK1 envelope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the wire format proposed in PR #63 review thread. Every serialized sketch binary is now wrapped in the ASK1 envelope: [ b"ASK1" | version:u8 | kind_id_len:u8 | kind_id:bytes | msgpack_payload ] The `kind_id` replaces the old bare magic-byte header: - Portable (cross-language): 1-byte kind_id — same ID constants as before - Native Rust: 2-byte kind_id — [type_discriminant, hasher_id] The ASK1 framing lifts the 256-sketch-type ceiling while keeping the same self-describing semantics. The 4-byte sentinel (b"ASK1") is unambiguous since no msgpack value starts with 0x41. All three original constraints hold: - API unchanged (same serialize_to_bytes/from_msgpack signatures) - Round trips verified (432 tests, all pass) - Different sketch types/modes produce different kind_ids Co-Authored-By: Claude Sonnet 4.6 (1M context) --- docs/msgpack-magic-ids.md | 60 +++++++------ src/message_pack_format/magic_ids.rs | 87 +++++++++++++++--- .../portable/countminsketch.rs | 37 ++++---- .../portable/countminsketch_topk.rs | 28 +++--- .../portable/countsketch.rs | 23 +++-- src/message_pack_format/portable/ddsketch.rs | 37 +++++--- .../portable/delta_set_aggregator.rs | 23 +++-- src/message_pack_format/portable/hll.rs | 20 +++-- src/message_pack_format/portable/hydra_kll.rs | 28 +++--- src/message_pack_format/portable/kll.rs | 28 +++--- .../portable/set_aggregator.rs | 32 ++++--- src/sketch_framework/hydra.rs | 25 +++--- src/sketch_framework/univmon.rs | 25 +++--- src/sketches/countminsketch.rs | 65 +++++++------- src/sketches/countsketch.rs | 65 +++++++------- src/sketches/countsketch_topk.rs | 27 +++--- src/sketches/ddsketch.rs | 27 +++--- src/sketches/hll.rs | 88 ++++++++++--------- src/sketches/kll.rs | 25 +++--- src/sketches/kll_dynamic.rs | 30 +++---- src/sketches/kmv.rs | 25 +++--- 21 files changed, 474 insertions(+), 331 deletions(-) diff --git a/docs/msgpack-magic-ids.md b/docs/msgpack-magic-ids.md index 1a9037b..524b00d 100644 --- a/docs/msgpack-magic-ids.md +++ b/docs/msgpack-magic-ids.md @@ -1,29 +1,32 @@ -# MessagePack Magic IDs +# ASK1 Sketch Wire Format -Every serialised sketch binary produced by this library starts with a single -type-discriminant byte called the **magic ID**. Reading the first byte of any -blob is enough to identify what sketch type it contains — no full decode -required. +Every serialised sketch binary produced by this library is wrapped in the +**ASK1 envelope**, a self-describing header that carries the sketch's +type discriminant without a fixed-size ceiling on the number of types. -**Portable** (cross-language): ``` -┌─────────────┬───────────────────────────────────────┐ -│ magic_id: u8 │ rmp_serde / msgpack payload … │ -└─────────────┴───────────────────────────────────────┘ +┌────────────┬────────────┬──────────────────┬───────────────────────┬──────────────────────┐ +│ b"ASK1": 4B │ version: u8 │ kind_id_len: u8 │ kind_id: [kind_id_len] │ msgpack payload … │ +└────────────┴────────────┴──────────────────┴───────────────────────┴──────────────────────┘ ``` -**Native** (Rust-internal): -``` -┌─────────────┬──────────────┬───────────────────────────────────┐ -│ magic_id: u8 │ hasher_id: u8 │ rmp_serde / msgpack payload … │ -└─────────────┴──────────────┴───────────────────────────────────┘ -``` +| Field | Value | Notes | +|-------|-------|-------| +| `b"ASK1"` | `0x41 0x53 0x4B 0x31` | 4-byte ASCII sentinel, not a valid msgpack prefix | +| `version` | `0x01` | Increment only if the envelope layout changes | +| `kind_id_len` | 1 or 2 | Number of `kind_id` bytes (1 for portable, 2 for native) | +| `kind_id` | see tables below | Canonical big-endian, no leading zero bytes | +| payload | msgpack bytes | Compact (array) or named (map) depending on sketch type | + +**Portable** sketches use a 1-byte `kind_id` (`kind_id_len = 1`). +**Native** Rust sketches use a 2-byte `kind_id` (`kind_id_len = 2`): first byte +is the type/mode discriminant, second byte is the hasher ID (`HASHER_*`). -The `hasher_id` byte is `0xFF` (`HASHER_UNKNOWN`) for types without an `H` parameter. -Custom hashers that do not register an ID also store `0xFF` — the mismatch check -is skipped on both sides when either value is `0xFF`. +The `hasher_id` second byte is `0xFF` (`HASHER_UNKNOWN`) for types without an `H` +parameter. Custom hashers that do not register an ID also store `0xFF` — the mismatch +check is skipped on both sides when either value is `0xFF`. -Magic IDs are **stable** — once assigned, a value is never reused or +Kind IDs are **stable** — once assigned, a value is never reused or reassigned. Adding a new sketch type requires a new constant; removing or repurposing an existing constant is a **breaking protocol change**. @@ -95,10 +98,11 @@ bytes are produced by `KLL::serialize_to_bytes` and therefore carry the native `0x87` prefix. The portable round-trip is: ``` -KllSketch::to_msgpack() → [ 0x06 | msgpack([k, [0x87 | raw_kll_bytes]]) ] -KllSketch::from_msgpack() → strips 0x06, decodes struct, - calls KLL::deserialize_from_bytes([0x87 | …]) - → strips 0x87, decodes KLL +KllSketch::to_msgpack() → [ ASK1 | v=1 | len=1 | 0x06 | msgpack([k, [ASK1|v=1|len=2|0x8a|0xff|raw_kll]]) ] +KllSketch::from_msgpack() → decode_wrapper → kind_id=[0x06], payload=msgpack struct + → decode struct: k + sketch_bytes + → KLL::deserialize_from_bytes(sketch_bytes) + → decode_wrapper → kind_id=[0x8a, 0xff], payload=raw_kll ``` The same pattern applies to `HydraKllSketch` (`0x07`), which contains a grid @@ -113,9 +117,13 @@ of KLL cells. 2. If the type has a **portable** wire format shared with Go, also add the constant to `sketchlib-go/wire/asapmsgpack/magic_ids.go` with the same value, and update `SerializeMsgpack` / `DeserializeMsgpack` in the - corresponding Go sketch package. + corresponding Go sketch package to use `asapmsgpack.EncodeWrapper` / + `asapmsgpack.DecodeWrapper`. 3. Implement `MessagePackCodec::to_msgpack` / `from_msgpack` (or - `serialize_to_bytes` / `deserialize_from_bytes` for native types) to - prepend / validate the new constant. + `serialize_to_bytes` / `deserialize_from_bytes` for native types) using + `magic_ids::encode_wrapper` / `magic_ids::decode_wrapper`. + - Portable: `kind_id = &[MAGIC_CONSTANT]` (1 byte) + - Native with hasher: `kind_id = &[TYPE_BYTE, H::hasher_magic_id()]` (2 bytes) + - Native without hasher: `kind_id = &[TYPE_BYTE, HASHER_UNKNOWN]` (2 bytes) 4. Add or update round-trip tests. 5. Update this document. diff --git a/src/message_pack_format/magic_ids.rs b/src/message_pack_format/magic_ids.rs index d1555b6..cd7a3b7 100644 --- a/src/message_pack_format/magic_ids.rs +++ b/src/message_pack_format/magic_ids.rs @@ -1,19 +1,84 @@ -//! Magic-ID constants for the portable MessagePack wire format. +//! Magic-ID constants and wrapper encoding for the ASAP sketch wire format. //! -//! Every serialized binary produced by [`crate::message_pack_format::MessagePackCodec`] -//! is prefixed with a single byte that identifies the sketch type, analogous to -//! how Prometheus uses magic bytes to discriminate metric types. -//! -//! The prefix layout is: +//! Every serialized binary produced by this library is wrapped in the **ASK1 +//! envelope**, a self-describing header that identifies the sketch type and +//! reserves space for future metadata without a fixed-size ceiling: //! //! ```text -//! [ magic_id: u8 | ] +//! [ b"ASK1" | version: u8 | kind_id_len: u8 | kind_id: [u8; kind_id_len] | ] //! ``` //! -//! Magic IDs are stable across versions. Adding a new sketch type requires a -//! new constant here; removing or repurposing an existing constant is a -//! breaking protocol change. The Go mirror of this table lives in -//! `sketchlib-go/wire/asapmsgpack/magic_ids.go`. +//! * `b"ASK1"` — 4-byte ASCII sentinel; unambiguously not a msgpack value. +//! * `version` — envelope layout version; currently `0x01`. +//! * `kind_id_len + kind_id` — variable-length sketch discriminant encoded as +//! canonical unsigned big-endian with no leading zero bytes. For current +//! sketch types this is 1–2 bytes; future additions can use more without a +//! protocol change. +//! +//! **Portable** sketches (cross-language, shared with `sketchlib-go`) use a +//! 1-byte `kind_id` drawn from the `0x01–0x09` range. +//! +//! **Native** Rust sketches use a 2-byte `kind_id`: first byte is the family / +//! mode discriminant (`0x81–0x8e`); second byte is the hasher ID (`HASHER_*`). +//! +//! Magic IDs are **stable** — once assigned, a value is never reused. Adding +//! a new sketch type requires a new constant here; removing or repurposing an +//! existing constant is a **breaking protocol change**. The Go mirror of this +//! table lives in `sketchlib-go/wire/asapmsgpack/magic_ids.go`. + +// ── Envelope constants ──────────────────────────────────────────────────────── + +/// 4-byte ASCII sentinel that opens every ASAP sketch binary. +pub const WRAPPER_MAGIC: &[u8; 4] = b"ASK1"; + +/// Envelope layout version stored in byte 4. Increment if the header +/// structure (field order, field semantics) ever changes. +pub const WRAPPER_VERSION: u8 = 0x01; + +/// Prepend the ASK1 envelope to `payload` and return the complete binary. +/// +/// `kind_id` identifies the sketch type (1–2 bytes for all current types). +pub fn encode_wrapper(kind_id: &[u8], payload: &[u8]) -> Vec { + let mut out = Vec::with_capacity(4 + 1 + 1 + kind_id.len() + payload.len()); + out.extend_from_slice(WRAPPER_MAGIC); + out.push(WRAPPER_VERSION); + out.push(kind_id.len() as u8); + out.extend_from_slice(kind_id); + out.extend_from_slice(payload); + out +} + +/// Strip the ASK1 envelope from `bytes` and return `(kind_id, payload)`. +/// +/// Returns `Err(String)` on any structural mismatch so callers can convert to +/// their own error type. +pub fn decode_wrapper(bytes: &[u8]) -> Result<(&[u8], &[u8]), String> { + if bytes.len() < 7 { + return Err(format!( + "ASK1 wrapper: too short ({} bytes, need ≥7)", + bytes.len() + )); + } + if &bytes[0..4] != WRAPPER_MAGIC { + return Err(format!( + "ASK1 wrapper: bad magic {:?}, expected b\"ASK1\"", + &bytes[0..4] + )); + } + let version = bytes[4]; + if version != WRAPPER_VERSION { + return Err(format!("ASK1 wrapper: unsupported version 0x{version:02x}")); + } + let kind_id_len = bytes[5] as usize; + let header_end = 6 + kind_id_len; + if bytes.len() < header_end { + return Err(format!( + "ASK1 wrapper: kind_id_len={kind_id_len} but only {} bytes available after offset 6", + bytes.len().saturating_sub(6) + )); + } + Ok((&bytes[6..header_end], &bytes[header_end..])) +} /// HLL sketch (all variants: Regular, Datafusion, Hip). pub const HLL: u8 = 0x01; diff --git a/src/message_pack_format/portable/countminsketch.rs b/src/message_pack_format/portable/countminsketch.rs index 1eafed7..52b8bbf 100644 --- a/src/message_pack_format/portable/countminsketch.rs +++ b/src/message_pack_format/portable/countminsketch.rs @@ -442,27 +442,32 @@ impl MessagePackCodec for CountMinSketch { rows: self.rows, cols: self.cols, }; - let mut out = vec![magic_ids::COUNT_MIN_SKETCH]; - out.extend(rmp_serde::to_vec(&wire)?); - Ok(out) + let payload = rmp_serde::to_vec(&wire)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::COUNT_MIN_SKETCH], + &payload, + )) } fn from_msgpack(bytes: &[u8]) -> Result { - match bytes.first() { - Some(&magic_ids::COUNT_MIN_SKETCH) => { - let wire: CountMinSketchWire = rmp_serde::from_slice(&bytes[1..])?; - let backend = sketchlib_cms_from_matrix(wire.rows, wire.cols, &wire.sketch); - Ok(Self { - rows: wire.rows, - cols: wire.cols, - backend, - }) - } - other => Err(MsgPackError::BadMagicId { + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(|_| MsgPackError::BadMagicId { + expected: magic_ids::COUNT_MIN_SKETCH, + got: bytes.first().copied(), + })?; + if kind_id != [magic_ids::COUNT_MIN_SKETCH] { + return Err(MsgPackError::BadMagicId { expected: magic_ids::COUNT_MIN_SKETCH, - got: other.copied(), - }), + got: kind_id.first().copied(), + }); } + let wire: CountMinSketchWire = rmp_serde::from_slice(payload)?; + let backend = sketchlib_cms_from_matrix(wire.rows, wire.cols, &wire.sketch); + Ok(Self { + rows: wire.rows, + cols: wire.cols, + backend, + }) } } diff --git a/src/message_pack_format/portable/countminsketch_topk.rs b/src/message_pack_format/portable/countminsketch_topk.rs index 06e3089..c7d3ff0 100644 --- a/src/message_pack_format/portable/countminsketch_topk.rs +++ b/src/message_pack_format/portable/countminsketch_topk.rs @@ -297,21 +297,25 @@ impl MessagePackCodec for CountMinSketchWithHeap { topk_heap: self.topk_heap_items(), heap_size: self.heap_size, }; - let mut out = vec![magic_ids::COUNT_MIN_SKETCH_WITH_HEAP]; - out.extend(rmp_serde::to_vec(&wire)?); - Ok(out) + let payload = rmp_serde::to_vec(&wire)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::COUNT_MIN_SKETCH_WITH_HEAP], + &payload, + )) } fn from_msgpack(bytes: &[u8]) -> Result { - let payload = match bytes.first() { - Some(&magic_ids::COUNT_MIN_SKETCH_WITH_HEAP) => &bytes[1..], - other => { - return Err(MsgPackError::BadMagicId { - expected: magic_ids::COUNT_MIN_SKETCH_WITH_HEAP, - got: other.copied(), - }); - } - }; + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(|_| MsgPackError::BadMagicId { + expected: magic_ids::COUNT_MIN_SKETCH_WITH_HEAP, + got: bytes.first().copied(), + })?; + if kind_id != [magic_ids::COUNT_MIN_SKETCH_WITH_HEAP] { + return Err(MsgPackError::BadMagicId { + expected: magic_ids::COUNT_MIN_SKETCH_WITH_HEAP, + got: kind_id.first().copied(), + }); + } let wire: CountMinSketchWithHeapWire = rmp_serde::from_slice(payload)?; let mut sorted_topk_heap = wire.topk_heap; diff --git a/src/message_pack_format/portable/countsketch.rs b/src/message_pack_format/portable/countsketch.rs index a57a96c..fb20d96 100644 --- a/src/message_pack_format/portable/countsketch.rs +++ b/src/message_pack_format/portable/countsketch.rs @@ -434,19 +434,26 @@ impl CountSketch { impl MessagePackCodec for CountSketch { fn to_msgpack(&self) -> Result, MsgPackError> { - let mut out = vec![magic_ids::COUNT_SKETCH]; - out.extend(rmp_serde::to_vec(self)?); - Ok(out) + let payload = rmp_serde::to_vec(self)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::COUNT_SKETCH], + &payload, + )) } fn from_msgpack(bytes: &[u8]) -> Result { - match bytes.first() { - Some(&magic_ids::COUNT_SKETCH) => Ok(rmp_serde::from_slice(&bytes[1..])?), - other => Err(MsgPackError::BadMagicId { + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(|_| MsgPackError::BadMagicId { expected: magic_ids::COUNT_SKETCH, - got: other.copied(), - }), + got: bytes.first().copied(), + })?; + if kind_id != [magic_ids::COUNT_SKETCH] { + return Err(MsgPackError::BadMagicId { + expected: magic_ids::COUNT_SKETCH, + got: kind_id.first().copied(), + }); } + Ok(rmp_serde::from_slice(payload)?) } } diff --git a/src/message_pack_format/portable/ddsketch.rs b/src/message_pack_format/portable/ddsketch.rs index bc0b425..71aa67b 100644 --- a/src/message_pack_format/portable/ddsketch.rs +++ b/src/message_pack_format/portable/ddsketch.rs @@ -403,19 +403,23 @@ impl DdSketch { impl MessagePackCodec for DdSketch { fn to_msgpack(&self) -> Result, MsgPackError> { - let mut out = vec![magic_ids::DD_SKETCH]; - out.extend(rmp_serde::to_vec(self)?); - Ok(out) + let payload = rmp_serde::to_vec(self)?; + Ok(magic_ids::encode_wrapper(&[magic_ids::DD_SKETCH], &payload)) } fn from_msgpack(bytes: &[u8]) -> Result { - match bytes.first() { - Some(&magic_ids::DD_SKETCH) => Ok(rmp_serde::from_slice(&bytes[1..])?), - other => Err(MsgPackError::BadMagicId { + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(|_| MsgPackError::BadMagicId { expected: magic_ids::DD_SKETCH, - got: other.copied(), - }), + got: bytes.first().copied(), + })?; + if kind_id != [magic_ids::DD_SKETCH] { + return Err(MsgPackError::BadMagicId { + expected: magic_ids::DD_SKETCH, + got: kind_id.first().copied(), + }); } + Ok(rmp_serde::from_slice(payload)?) } } @@ -548,20 +552,25 @@ mod tests { /// element count so the bytes stay parity-aligned with the Go /// reference implementation. /// - /// bytes[0] is now the magic-ID prefix (0x05 = DD_SKETCH); the msgpack - /// payload starts at bytes[1]. + /// The binary is wrapped in the ASK1 envelope; the payload starts after + /// the header (b"ASK1" + version + kind_id_len + kind_id). #[test] fn test_msgpack_is_three_element_array() { use crate::message_pack_format::magic_ids; let sk = DdSketch::from_raw(0.01, vec![1, 2, 3], -2); let bytes = sk.to_msgpack().unwrap(); - assert_eq!(bytes[0], magic_ids::DD_SKETCH, "expected magic-ID byte"); + let (kind_id, payload) = magic_ids::decode_wrapper(&bytes).expect("ASK1 header"); + assert_eq!( + kind_id, + [magic_ids::DD_SKETCH], + "expected DD_SKETCH kind_id" + ); // rmp compact encoding of the payload leads with an array marker. // A fixarray of length 3 is the single byte 0x93 (0b1001_0011). assert_eq!( - bytes[1], 0x93, - "expected a 3-element msgpack fixarray (0x93) at bytes[1], got {:#04x}", - bytes[1] + payload[0], 0x93, + "expected a 3-element msgpack fixarray (0x93) at payload[0], got {:#04x}", + payload[0] ); } diff --git a/src/message_pack_format/portable/delta_set_aggregator.rs b/src/message_pack_format/portable/delta_set_aggregator.rs index 6a87dda..635dedf 100644 --- a/src/message_pack_format/portable/delta_set_aggregator.rs +++ b/src/message_pack_format/portable/delta_set_aggregator.rs @@ -20,19 +20,26 @@ pub struct DeltaResult { impl MessagePackCodec for DeltaResult { fn to_msgpack(&self) -> Result, MsgPackError> { - let mut out = vec![magic_ids::DELTA_RESULT]; - out.extend(rmp_serde::to_vec(self)?); - Ok(out) + let payload = rmp_serde::to_vec(self)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::DELTA_RESULT], + &payload, + )) } fn from_msgpack(bytes: &[u8]) -> Result { - match bytes.first() { - Some(&magic_ids::DELTA_RESULT) => Ok(rmp_serde::from_slice(&bytes[1..])?), - other => Err(MsgPackError::BadMagicId { + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(|_| MsgPackError::BadMagicId { expected: magic_ids::DELTA_RESULT, - got: other.copied(), - }), + got: bytes.first().copied(), + })?; + if kind_id != [magic_ids::DELTA_RESULT] { + return Err(MsgPackError::BadMagicId { + expected: magic_ids::DELTA_RESULT, + got: kind_id.first().copied(), + }); } + Ok(rmp_serde::from_slice(payload)?) } } diff --git a/src/message_pack_format/portable/hll.rs b/src/message_pack_format/portable/hll.rs index d87e7ab..5bdf722 100644 --- a/src/message_pack_format/portable/hll.rs +++ b/src/message_pack_format/portable/hll.rs @@ -430,19 +430,23 @@ fn read_uvarint(buf: &[u8]) -> Option<(u64, usize)> { impl MessagePackCodec for HllSketch { fn to_msgpack(&self) -> Result, MsgPackError> { - let mut out = vec![magic_ids::HLL]; - out.extend(rmp_serde::to_vec(self)?); - Ok(out) + let payload = rmp_serde::to_vec(self)?; + Ok(magic_ids::encode_wrapper(&[magic_ids::HLL], &payload)) } fn from_msgpack(bytes: &[u8]) -> Result { - match bytes.first() { - Some(&magic_ids::HLL) => Ok(rmp_serde::from_slice(&bytes[1..])?), - other => Err(MsgPackError::BadMagicId { + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(|_| MsgPackError::BadMagicId { expected: magic_ids::HLL, - got: other.copied(), - }), + got: bytes.first().copied(), + })?; + if kind_id != [magic_ids::HLL] { + return Err(MsgPackError::BadMagicId { + expected: magic_ids::HLL, + got: kind_id.first().copied(), + }); } + Ok(rmp_serde::from_slice(payload)?) } } diff --git a/src/message_pack_format/portable/hydra_kll.rs b/src/message_pack_format/portable/hydra_kll.rs index b12ae69..5b4a802 100644 --- a/src/message_pack_format/portable/hydra_kll.rs +++ b/src/message_pack_format/portable/hydra_kll.rs @@ -157,24 +157,28 @@ impl MessagePackCodec for HydraKllSketch { cols: self.cols, sketches, }; - let mut out = vec![magic_ids::HYDRA_KLL_SKETCH]; - out.extend(rmp_serde::to_vec(&wire)?); - Ok(out) + let payload = rmp_serde::to_vec(&wire)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::HYDRA_KLL_SKETCH], + &payload, + )) } fn from_msgpack(bytes: &[u8]) -> Result { use crate::sketches::kll::KLL; use rmp_serde::decode::Error as RmpDecodeError; - let payload = match bytes.first() { - Some(&magic_ids::HYDRA_KLL_SKETCH) => &bytes[1..], - other => { - return Err(MsgPackError::BadMagicId { - expected: magic_ids::HYDRA_KLL_SKETCH, - got: other.copied(), - }); - } - }; + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(|_| MsgPackError::BadMagicId { + expected: magic_ids::HYDRA_KLL_SKETCH, + got: bytes.first().copied(), + })?; + if kind_id != [magic_ids::HYDRA_KLL_SKETCH] { + return Err(MsgPackError::BadMagicId { + expected: magic_ids::HYDRA_KLL_SKETCH, + got: kind_id.first().copied(), + }); + } let wire: HydraKllSketchWire = rmp_serde::from_slice(payload)?; if wire.sketches.len() != wire.rows { diff --git a/src/message_pack_format/portable/kll.rs b/src/message_pack_format/portable/kll.rs index 045fb61..c065ea2 100644 --- a/src/message_pack_format/portable/kll.rs +++ b/src/message_pack_format/portable/kll.rs @@ -347,21 +347,25 @@ impl MessagePackCodec for KllSketch { k: self.k, sketch_bytes: self.sketch_bytes(), }; - let mut out = vec![magic_ids::KLL_SKETCH]; - out.extend(rmp_serde::to_vec(&wire)?); - Ok(out) + let payload = rmp_serde::to_vec(&wire)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::KLL_SKETCH], + &payload, + )) } fn from_msgpack(bytes: &[u8]) -> Result { - let payload = match bytes.first() { - Some(&magic_ids::KLL_SKETCH) => &bytes[1..], - other => { - return Err(MsgPackError::BadMagicId { - expected: magic_ids::KLL_SKETCH, - got: other.copied(), - }); - } - }; + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(|_| MsgPackError::BadMagicId { + expected: magic_ids::KLL_SKETCH, + got: bytes.first().copied(), + })?; + if kind_id != [magic_ids::KLL_SKETCH] { + return Err(MsgPackError::BadMagicId { + expected: magic_ids::KLL_SKETCH, + got: kind_id.first().copied(), + }); + } let wire: KllSketchData = rmp_serde::from_slice(payload)?; let backend = KLL::deserialize_from_bytes(&wire.sketch_bytes)?; Ok(Self { k: wire.k, backend }) diff --git a/src/message_pack_format/portable/set_aggregator.rs b/src/message_pack_format/portable/set_aggregator.rs index 85b2d8a..7ae4ac4 100644 --- a/src/message_pack_format/portable/set_aggregator.rs +++ b/src/message_pack_format/portable/set_aggregator.rs @@ -81,21 +81,25 @@ impl MessagePackCodec for SetAggregator { let wrapper = StringSetRef { values: &self.values, }; - let mut out = vec![magic_ids::SET_AGGREGATOR]; - out.extend(rmp_serde::to_vec(&wrapper)?); - Ok(out) + let payload = rmp_serde::to_vec(&wrapper)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::SET_AGGREGATOR], + &payload, + )) } fn from_msgpack(bytes: &[u8]) -> Result { - let payload = match bytes.first() { - Some(&magic_ids::SET_AGGREGATOR) => &bytes[1..], - other => { - return Err(MsgPackError::BadMagicId { - expected: magic_ids::SET_AGGREGATOR, - got: other.copied(), - }); - } - }; + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(|_| MsgPackError::BadMagicId { + expected: magic_ids::SET_AGGREGATOR, + got: bytes.first().copied(), + })?; + if kind_id != [magic_ids::SET_AGGREGATOR] { + return Err(MsgPackError::BadMagicId { + expected: magic_ids::SET_AGGREGATOR, + got: kind_id.first().copied(), + }); + } let wrapper: StringSetOwned = rmp_serde::from_slice(payload)?; Ok(Self { values: wrapper.values, @@ -165,9 +169,9 @@ mod tests { let mut sa = SetAggregator::new(); sa.update("a"); let bytes = sa.to_msgpack().unwrap(); - // Strip the leading magic byte before raw-decoding the payload. + let (_, payload) = magic_ids::decode_wrapper(&bytes).expect("ASK1 header"); let decoded: StringSet = - rmp_serde::from_slice(&bytes[1..]).expect("should decode as StringSet { values: ... }"); + rmp_serde::from_slice(payload).expect("should decode as StringSet { values: ... }"); assert!(decoded.values.contains("a")); } } diff --git a/src/sketch_framework/hydra.rs b/src/sketch_framework/hydra.rs index bcd0ecc..8f9f627 100644 --- a/src/sketch_framework/hydra.rs +++ b/src/sketch_framework/hydra.rs @@ -160,27 +160,28 @@ impl Hydra { self.query_key(key, &HydraQuery::Cdf(threshold)) } - /// Serializes the Hydra sketch (including all counters) into MessagePack bytes, - /// prefixed with the [`crate::message_pack_format::magic_ids::NATIVE_HYDRA`] magic byte. + /// Serializes the Hydra sketch into ASK1-wrapped MessagePack bytes. + /// kind_id: `[NATIVE_HYDRA, HASHER_UNKNOWN]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; - let mut out = vec![magic_ids::NATIVE_HYDRA, magic_ids::HASHER_UNKNOWN]; - out.extend(to_vec_named(self)?); - Ok(out) + let payload = to_vec_named(self)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::NATIVE_HYDRA, magic_ids::HASHER_UNKNOWN], + &payload, + )) } /// Deserializes a Hydra sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { use crate::message_pack_format::magic_ids; - match bytes { - [id, _hasher, rest @ ..] if *id == magic_ids::NATIVE_HYDRA => from_slice(rest), + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(RmpDecodeError::Uncategorized)?; + match kind_id { + [id, _hasher] if *id == magic_ids::NATIVE_HYDRA => from_slice(payload), _ => Err(RmpDecodeError::Uncategorized(format!( - "Hydra magic-ID mismatch: expected 0x{:02x}, got {:?}", + "Hydra kind_id mismatch: expected [0x{:02x}, hasher], got {:?}", magic_ids::NATIVE_HYDRA, - bytes - .first() - .map(|b| format!("0x{b:02x}")) - .unwrap_or_else(|| "empty buffer".to_string()) + kind_id ))), } } diff --git a/src/sketch_framework/univmon.rs b/src/sketch_framework/univmon.rs index 8406ceb..b7caf59 100644 --- a/src/sketch_framework/univmon.rs +++ b/src/sketch_framework/univmon.rs @@ -265,27 +265,28 @@ impl UnivMon { &mut self.hh_layers[layer] } - /// Serializes the UnivMon sketch into MessagePack bytes, prefixed with the - /// [`crate::message_pack_format::magic_ids::NATIVE_UNIVMON`] magic byte. + /// Serializes the UnivMon sketch into ASK1-wrapped MessagePack bytes. + /// kind_id: `[NATIVE_UNIVMON, HASHER_UNKNOWN]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; - let mut out = vec![magic_ids::NATIVE_UNIVMON, magic_ids::HASHER_UNKNOWN]; - out.extend(to_vec_named(self)?); - Ok(out) + let payload = to_vec_named(self)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::NATIVE_UNIVMON, magic_ids::HASHER_UNKNOWN], + &payload, + )) } /// Deserializes a UnivMon sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { use crate::message_pack_format::magic_ids; - match bytes { - [id, _hasher, rest @ ..] if *id == magic_ids::NATIVE_UNIVMON => from_slice(rest), + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(RmpDecodeError::Uncategorized)?; + match kind_id { + [id, _hasher] if *id == magic_ids::NATIVE_UNIVMON => from_slice(payload), _ => Err(RmpDecodeError::Uncategorized(format!( - "UnivMon magic-ID mismatch: expected 0x{:02x}, got {:?}", + "UnivMon kind_id mismatch: expected [0x{:02x}, hasher], got {:?}", magic_ids::NATIVE_UNIVMON, - bytes - .first() - .map(|b| format!("0x{b:02x}")) - .unwrap_or_else(|| "empty buffer".to_string()) + kind_id ))), } } diff --git a/src/sketches/countminsketch.rs b/src/sketches/countminsketch.rs index 49c32a8..caa628d 100644 --- a/src/sketches/countminsketch.rs +++ b/src/sketches/countminsketch.rs @@ -266,13 +266,15 @@ impl CountMin { // Wire layout: [ mode_id: u8 | hasher_id: u8 | ] impl CountMin { - /// Serializes the sketch into MessagePack bytes. - /// Header: `[NATIVE_COUNT_MIN_REGULAR, hasher_id]`. + /// Serializes the sketch into ASK1-wrapped MessagePack bytes. + /// kind_id: `[NATIVE_COUNT_MIN_REGULAR, hasher_id]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; - let mut out = vec![magic_ids::NATIVE_COUNT_MIN_REGULAR, H::hasher_magic_id()]; - out.extend(to_vec_named(self)?); - Ok(out) + let payload = to_vec_named(self)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::NATIVE_COUNT_MIN_REGULAR, H::hasher_magic_id()], + &payload, + )) } } @@ -280,31 +282,32 @@ impl Deserialize<'de>, H: SketchHasher> CountMin Result { use crate::message_pack_format::magic_ids; - match bytes { - [id, hasher, rest @ ..] if *id == magic_ids::NATIVE_COUNT_MIN_REGULAR => { + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(RmpDecodeError::Uncategorized)?; + match kind_id { + [id, hasher] if *id == magic_ids::NATIVE_COUNT_MIN_REGULAR => { magic_ids::check_hasher_id::(*hasher)?; - from_slice(rest) + from_slice(payload) } _ => Err(RmpDecodeError::Uncategorized(format!( - "CountMin magic-ID mismatch: expected 0x{:02x}, got {:?}", + "CountMin kind_id mismatch: expected [0x{:02x}, hasher], got {:?}", magic_ids::NATIVE_COUNT_MIN_REGULAR, - bytes - .first() - .map(|b| format!("0x{b:02x}")) - .unwrap_or_else(|| "empty buffer".to_string()) + kind_id ))), } } } impl CountMin { - /// Serializes the sketch into MessagePack bytes. - /// Header: `[NATIVE_COUNT_MIN_FAST, hasher_id]`. + /// Serializes the sketch into ASK1-wrapped MessagePack bytes. + /// kind_id: `[NATIVE_COUNT_MIN_FAST, hasher_id]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; - let mut out = vec![magic_ids::NATIVE_COUNT_MIN_FAST, H::hasher_magic_id()]; - out.extend(to_vec_named(self)?); - Ok(out) + let payload = to_vec_named(self)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::NATIVE_COUNT_MIN_FAST, H::hasher_magic_id()], + &payload, + )) } } @@ -312,18 +315,17 @@ impl Deserialize<'de>, H: SketchHasher> CountMin Result { use crate::message_pack_format::magic_ids; - match bytes { - [id, hasher, rest @ ..] if *id == magic_ids::NATIVE_COUNT_MIN_FAST => { + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(RmpDecodeError::Uncategorized)?; + match kind_id { + [id, hasher] if *id == magic_ids::NATIVE_COUNT_MIN_FAST => { magic_ids::check_hasher_id::(*hasher)?; - from_slice(rest) + from_slice(payload) } _ => Err(RmpDecodeError::Uncategorized(format!( - "CountMin magic-ID mismatch: expected 0x{:02x}, got {:?}", + "CountMin kind_id mismatch: expected [0x{:02x}, hasher], got {:?}", magic_ids::NATIVE_COUNT_MIN_FAST, - bytes - .first() - .map(|b| format!("0x{b:02x}")) - .unwrap_or_else(|| "empty buffer".to_string()) + kind_id ))), } } @@ -1025,12 +1027,15 @@ mod tests { let encoded = sketch .serialize_to_bytes() .expect("serialize CountMin FastPath"); + let (kind_id, _) = magic_ids::decode_wrapper(&encoded).expect("ASK1 header"); assert_eq!( - encoded[0], - magic_ids::NATIVE_COUNT_MIN_FAST, - "wrong magic id" + kind_id, + [ + magic_ids::NATIVE_COUNT_MIN_FAST, + magic_ids::HASHER_DEFAULT_XX + ], + "wrong kind_id" ); - assert_eq!(encoded[1], magic_ids::HASHER_DEFAULT_XX, "wrong hasher id"); let decoded = CountMin::, FastPath>::deserialize_from_bytes(&encoded) .expect("deserialize CountMin FastPath"); diff --git a/src/sketches/countsketch.rs b/src/sketches/countsketch.rs index f50840e..14161ef 100644 --- a/src/sketches/countsketch.rs +++ b/src/sketches/countsketch.rs @@ -284,13 +284,15 @@ where S: MatrixStorage + Serialize, C: CountSketchCounter, { - /// Serializes the sketch into MessagePack bytes. - /// Header: `[NATIVE_COUNT_SKETCH_REGULAR, hasher_id]`. + /// Serializes the sketch into ASK1-wrapped MessagePack bytes. + /// kind_id: `[NATIVE_COUNT_SKETCH_REGULAR, hasher_id]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; - let mut out = vec![magic_ids::NATIVE_COUNT_SKETCH_REGULAR, H::hasher_magic_id()]; - out.extend(to_vec_named(self)?); - Ok(out) + let payload = to_vec_named(self)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::NATIVE_COUNT_SKETCH_REGULAR, H::hasher_magic_id()], + &payload, + )) } } @@ -302,18 +304,17 @@ where /// Deserializes a sketch from bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { use crate::message_pack_format::magic_ids; - match bytes { - [id, hasher, rest @ ..] if *id == magic_ids::NATIVE_COUNT_SKETCH_REGULAR => { + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(RmpDecodeError::Uncategorized)?; + match kind_id { + [id, hasher] if *id == magic_ids::NATIVE_COUNT_SKETCH_REGULAR => { magic_ids::check_hasher_id::(*hasher)?; - from_slice(rest) + from_slice(payload) } _ => Err(RmpDecodeError::Uncategorized(format!( - "Count magic-ID mismatch: expected 0x{:02x}, got {:?}", + "Count kind_id mismatch: expected [0x{:02x}, hasher], got {:?}", magic_ids::NATIVE_COUNT_SKETCH_REGULAR, - bytes - .first() - .map(|b| format!("0x{b:02x}")) - .unwrap_or_else(|| "empty buffer".to_string()) + kind_id ))), } } @@ -324,13 +325,15 @@ where S: MatrixStorage + Serialize, C: CountSketchCounter, { - /// Serializes the sketch into MessagePack bytes. - /// Header: `[NATIVE_COUNT_SKETCH_FAST, hasher_id]`. + /// Serializes the sketch into ASK1-wrapped MessagePack bytes. + /// kind_id: `[NATIVE_COUNT_SKETCH_FAST, hasher_id]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; - let mut out = vec![magic_ids::NATIVE_COUNT_SKETCH_FAST, H::hasher_magic_id()]; - out.extend(to_vec_named(self)?); - Ok(out) + let payload = to_vec_named(self)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::NATIVE_COUNT_SKETCH_FAST, H::hasher_magic_id()], + &payload, + )) } } @@ -342,18 +345,17 @@ where /// Deserializes a sketch from bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { use crate::message_pack_format::magic_ids; - match bytes { - [id, hasher, rest @ ..] if *id == magic_ids::NATIVE_COUNT_SKETCH_FAST => { + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(RmpDecodeError::Uncategorized)?; + match kind_id { + [id, hasher] if *id == magic_ids::NATIVE_COUNT_SKETCH_FAST => { magic_ids::check_hasher_id::(*hasher)?; - from_slice(rest) + from_slice(payload) } _ => Err(RmpDecodeError::Uncategorized(format!( - "Count magic-ID mismatch: expected 0x{:02x}, got {:?}", + "Count kind_id mismatch: expected [0x{:02x}, hasher], got {:?}", magic_ids::NATIVE_COUNT_SKETCH_FAST, - bytes - .first() - .map(|b| format!("0x{b:02x}")) - .unwrap_or_else(|| "empty buffer".to_string()) + kind_id ))), } } @@ -1173,12 +1175,15 @@ mod tests { let encoded = sketch .serialize_to_bytes() .expect("serialize Count FastPath"); + let (kind_id, _) = magic_ids::decode_wrapper(&encoded).expect("ASK1 header"); assert_eq!( - encoded[0], - magic_ids::NATIVE_COUNT_SKETCH_FAST, - "wrong magic id" + kind_id, + [ + magic_ids::NATIVE_COUNT_SKETCH_FAST, + magic_ids::HASHER_DEFAULT_XX + ], + "wrong kind_id" ); - assert_eq!(encoded[1], magic_ids::HASHER_DEFAULT_XX, "wrong hasher id"); let decoded = Count::, FastPath>::deserialize_from_bytes(&encoded) .expect("deserialize Count FastPath"); diff --git a/src/sketches/countsketch_topk.rs b/src/sketches/countsketch_topk.rs index fe14bf5..4798e9e 100644 --- a/src/sketches/countsketch_topk.rs +++ b/src/sketches/countsketch_topk.rs @@ -1106,30 +1106,31 @@ impl CountL2HH { compute_median_inline_f64(&mut lst[..]) } - /// Serializes the CountL2HH sketch into MessagePack bytes, prefixed with the - /// [`crate::message_pack_format::magic_ids::NATIVE_CMS_HEAP`] magic byte. + /// Serializes the CountL2HH sketch into ASK1-wrapped MessagePack bytes. + /// kind_id: `[NATIVE_CMS_HEAP, hasher_id]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; - let mut out = vec![magic_ids::NATIVE_CMS_HEAP, H::hasher_magic_id()]; - out.extend(to_vec_named(self)?); - Ok(out) + let payload = to_vec_named(self)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::NATIVE_CMS_HEAP, H::hasher_magic_id()], + &payload, + )) } /// Deserializes a CountL2HH sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { use crate::message_pack_format::magic_ids; - match bytes { - [id, hasher, rest @ ..] if *id == magic_ids::NATIVE_CMS_HEAP => { + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(RmpDecodeError::Uncategorized)?; + match kind_id { + [id, hasher] if *id == magic_ids::NATIVE_CMS_HEAP => { magic_ids::check_hasher_id::(*hasher)?; - from_slice(rest) + from_slice(payload) } _ => Err(RmpDecodeError::Uncategorized(format!( - "CountL2HH magic-ID mismatch: expected 0x{:02x}, got {:?}", + "CountL2HH kind_id mismatch: expected [0x{:02x}, hasher], got {:?}", magic_ids::NATIVE_CMS_HEAP, - bytes - .first() - .map(|b| format!("0x{b:02x}")) - .unwrap_or_else(|| "empty buffer".to_string()) + kind_id ))), } } diff --git a/src/sketches/ddsketch.rs b/src/sketches/ddsketch.rs index 21c392f..4204139 100644 --- a/src/sketches/ddsketch.rs +++ b/src/sketches/ddsketch.rs @@ -165,13 +165,15 @@ impl DDSketch { } } - /// Serializes the sketch to a MessagePack byte vector, prefixed with the - /// [`crate::message_pack_format::magic_ids::NATIVE_DD_SKETCH`] magic byte. + /// Serializes the sketch to ASK1-wrapped MessagePack bytes. + /// kind_id: `[NATIVE_DD_SKETCH, HASHER_UNKNOWN]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; - let mut out = vec![magic_ids::NATIVE_DD_SKETCH, magic_ids::HASHER_UNKNOWN]; - out.extend(to_vec_named(self)?); - Ok(out) + let payload = to_vec_named(self)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::NATIVE_DD_SKETCH, magic_ids::HASHER_UNKNOWN], + &payload, + )) } /// Deserializes a DDSketch from a MessagePack byte slice produced by @@ -185,19 +187,18 @@ impl DDSketch { /// to within the sketch's α relative-accuracy bound. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { use crate::message_pack_format::magic_ids; - match bytes { - [id, _hasher, rest @ ..] if *id == magic_ids::NATIVE_DD_SKETCH => { - let mut sk: Self = from_slice(rest)?; + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(RmpDecodeError::Uncategorized)?; + match kind_id { + [id, _hasher] if *id == magic_ids::NATIVE_DD_SKETCH => { + let mut sk: Self = from_slice(payload)?; sk.recompute_scalars_from_store(); Ok(sk) } _ => Err(RmpDecodeError::Uncategorized(format!( - "DDSketch magic-ID mismatch: expected 0x{:02x}, got {:?}", + "DDSketch kind_id mismatch: expected [0x{:02x}, hasher], got {:?}", magic_ids::NATIVE_DD_SKETCH, - bytes - .first() - .map(|b| format!("0x{b:02x}")) - .unwrap_or_else(|| "empty buffer".to_string()) + kind_id ))), } } diff --git a/src/sketches/hll.rs b/src/sketches/hll.rs index d2ac7ca..7dab5ef 100644 --- a/src/sketches/hll.rs +++ b/src/sketches/hll.rs @@ -166,13 +166,15 @@ impl impl HyperLogLogImpl { - /// Serializes the sketch into MessagePack bytes. - /// Header: `[NATIVE_HLL_CLASSIC, hasher_id]`. + /// Serializes the sketch into ASK1-wrapped MessagePack bytes. + /// kind_id: `[NATIVE_HLL_CLASSIC, hasher_id]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; - let mut out = vec![magic_ids::NATIVE_HLL_CLASSIC, H::hasher_magic_id()]; - out.extend(to_vec_named(self)?); - Ok(out) + let payload = to_vec_named(self)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::NATIVE_HLL_CLASSIC, H::hasher_magic_id()], + &payload, + )) } } @@ -182,18 +184,17 @@ impl Deserialize<'de>, H: SketchHasher> /// Deserializes a sketch from bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { use crate::message_pack_format::magic_ids; - match bytes { - [id, hasher, rest @ ..] if *id == magic_ids::NATIVE_HLL_CLASSIC => { + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(RmpDecodeError::Uncategorized)?; + match kind_id { + [id, hasher] if *id == magic_ids::NATIVE_HLL_CLASSIC => { magic_ids::check_hasher_id::(*hasher)?; - from_slice(rest) + from_slice(payload) } _ => Err(RmpDecodeError::Uncategorized(format!( - "HyperLogLogImpl magic-ID mismatch: expected 0x{:02x}, got {:?}", + "HyperLogLogImpl kind_id mismatch: expected [0x{:02x}, hasher], got {:?}", magic_ids::NATIVE_HLL_CLASSIC, - bytes - .first() - .map(|b| format!("0x{b:02x}")) - .unwrap_or_else(|| "empty buffer".to_string()) + kind_id ))), } } @@ -202,13 +203,15 @@ impl Deserialize<'de>, H: SketchHasher> impl HyperLogLogImpl { - /// Serializes the sketch into MessagePack bytes. - /// Header: `[NATIVE_HLL_ERTL_MLE, hasher_id]`. + /// Serializes the sketch into ASK1-wrapped MessagePack bytes. + /// kind_id: `[NATIVE_HLL_ERTL_MLE, hasher_id]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; - let mut out = vec![magic_ids::NATIVE_HLL_ERTL_MLE, H::hasher_magic_id()]; - out.extend(to_vec_named(self)?); - Ok(out) + let payload = to_vec_named(self)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::NATIVE_HLL_ERTL_MLE, H::hasher_magic_id()], + &payload, + )) } } @@ -218,18 +221,17 @@ impl Deserialize<'de>, H: SketchHasher> /// Deserializes a sketch from bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { use crate::message_pack_format::magic_ids; - match bytes { - [id, hasher, rest @ ..] if *id == magic_ids::NATIVE_HLL_ERTL_MLE => { + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(RmpDecodeError::Uncategorized)?; + match kind_id { + [id, hasher] if *id == magic_ids::NATIVE_HLL_ERTL_MLE => { magic_ids::check_hasher_id::(*hasher)?; - from_slice(rest) + from_slice(payload) } _ => Err(RmpDecodeError::Uncategorized(format!( - "HyperLogLogImpl magic-ID mismatch: expected 0x{:02x}, got {:?}", + "HyperLogLogImpl kind_id mismatch: expected [0x{:02x}, hasher], got {:?}", magic_ids::NATIVE_HLL_ERTL_MLE, - bytes - .first() - .map(|b| format!("0x{b:02x}")) - .unwrap_or_else(|| "empty buffer".to_string()) + kind_id ))), } } @@ -434,27 +436,28 @@ impl HyperLogLogHIPImpl { self.est as usize } - /// Serializes the sketch into MessagePack bytes. - /// Header: `[NATIVE_HLL_HIP, HASHER_DEFAULT_XX]` — HIP always uses DefaultXxHasher. + /// Serializes the sketch into ASK1-wrapped MessagePack bytes. + /// kind_id: `[NATIVE_HLL_HIP, HASHER_DEFAULT_XX]` — HIP always uses DefaultXxHasher. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; - let mut out = vec![magic_ids::NATIVE_HLL_HIP, magic_ids::HASHER_DEFAULT_XX]; - out.extend(to_vec_named(self)?); - Ok(out) + let payload = to_vec_named(self)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::NATIVE_HLL_HIP, magic_ids::HASHER_DEFAULT_XX], + &payload, + )) } /// Deserializes a sketch from bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { use crate::message_pack_format::magic_ids; - match bytes { - [id, _hasher, rest @ ..] if *id == magic_ids::NATIVE_HLL_HIP => from_slice(rest), + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(RmpDecodeError::Uncategorized)?; + match kind_id { + [id, _hasher] if *id == magic_ids::NATIVE_HLL_HIP => from_slice(payload), _ => Err(RmpDecodeError::Uncategorized(format!( - "HyperLogLogHIPImpl magic-ID mismatch: expected 0x{:02x}, got {:?}", + "HyperLogLogHIPImpl kind_id mismatch: expected [0x{:02x}, hasher], got {:?}", magic_ids::NATIVE_HLL_HIP, - bytes - .first() - .map(|b| format!("0x{b:02x}")) - .unwrap_or_else(|| "empty buffer".to_string()) + kind_id ))), } } @@ -766,10 +769,13 @@ mod tests { .serialize_to_bytes() .unwrap(); - assert_eq!(classic_bytes[0], magic_ids::NATIVE_HLL_CLASSIC); - assert_eq!(ertl_bytes[0], magic_ids::NATIVE_HLL_ERTL_MLE); + let (classic_kind_id, _) = magic_ids::decode_wrapper(&classic_bytes).expect("ASK1 header"); + let (ertl_kind_id, _) = magic_ids::decode_wrapper(&ertl_bytes).expect("ASK1 header"); + + assert_eq!(classic_kind_id[0], magic_ids::NATIVE_HLL_CLASSIC); + assert_eq!(ertl_kind_id[0], magic_ids::NATIVE_HLL_ERTL_MLE); // Sanity: they are different. - assert_ne!(classic_bytes[0], ertl_bytes[0]); + assert_ne!(classic_kind_id[0], ertl_kind_id[0]); } #[test] diff --git a/src/sketches/kll.rs b/src/sketches/kll.rs index 5b6c20f..e65a042 100644 --- a/src/sketches/kll.rs +++ b/src/sketches/kll.rs @@ -662,16 +662,18 @@ impl KLL { // -- Serialization ------------------------------------------------------- - /// Serializes the sketch to a MessagePack byte vector, prefixed with the - /// [`crate::message_pack_format::magic_ids::NATIVE_KLL`] magic byte. + /// Serializes the sketch into ASK1-wrapped MessagePack bytes. + /// kind_id: `[NATIVE_KLL, HASHER_UNKNOWN]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> where T: Serialize, { use crate::message_pack_format::magic_ids; - let mut out = vec![magic_ids::NATIVE_KLL, magic_ids::HASHER_UNKNOWN]; - out.extend(rmp_serde::to_vec(self)?); - Ok(out) + let payload = rmp_serde::to_vec(self)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::NATIVE_KLL, magic_ids::HASHER_UNKNOWN], + &payload, + )) } /// Deserializes a KLL sketch from a MessagePack byte slice produced by @@ -681,15 +683,14 @@ impl KLL { T: for<'de> Deserialize<'de>, { use crate::message_pack_format::magic_ids; - match bytes { - [id, _hasher, rest @ ..] if *id == magic_ids::NATIVE_KLL => rmp_serde::from_slice(rest), + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(rmp_serde::decode::Error::Uncategorized)?; + match kind_id { + [id, _hasher] if *id == magic_ids::NATIVE_KLL => rmp_serde::from_slice(payload), _ => Err(rmp_serde::decode::Error::Uncategorized(format!( - "KLL magic-ID mismatch: expected 0x{:02x}, got {:?}", + "KLL kind_id mismatch: expected [0x{:02x}, hasher], got {:?}", magic_ids::NATIVE_KLL, - bytes - .first() - .map(|b| format!("0x{b:02x}")) - .unwrap_or_else(|| "empty buffer".to_string()) + kind_id ))), } } diff --git a/src/sketches/kll_dynamic.rs b/src/sketches/kll_dynamic.rs index ca1a739..51759e3 100644 --- a/src/sketches/kll_dynamic.rs +++ b/src/sketches/kll_dynamic.rs @@ -353,16 +353,18 @@ impl KLLDynamic { self.items.len() } - /// Serialize the sketch into MessagePack bytes, prefixed with the - /// [`crate::message_pack_format::magic_ids::NATIVE_KLL_DYNAMIC`] magic byte. + /// Serialize the sketch into ASK1-wrapped MessagePack bytes. + /// kind_id: `[NATIVE_KLL_DYNAMIC, HASHER_UNKNOWN]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> where T: Serialize, { use crate::message_pack_format::magic_ids; - let mut out = vec![magic_ids::NATIVE_KLL_DYNAMIC, magic_ids::HASHER_UNKNOWN]; - out.extend(rmp_serde::to_vec(self)?); - Ok(out) + let payload = rmp_serde::to_vec(self)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::NATIVE_KLL_DYNAMIC, magic_ids::HASHER_UNKNOWN], + &payload, + )) } /// Deserialize a sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. @@ -371,20 +373,18 @@ impl KLLDynamic { T: for<'de> Deserialize<'de>, { use crate::message_pack_format::magic_ids; - match bytes { - [id, _hasher, rest @ ..] if *id == magic_ids::NATIVE_KLL_DYNAMIC => { - rmp_serde::from_slice(rest).map(|mut sketch: KLLDynamic| { + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(rmp_serde::decode::Error::Uncategorized)?; + match kind_id { + [id, _hasher] if *id == magic_ids::NATIVE_KLL_DYNAMIC => rmp_serde::from_slice(payload) + .map(|mut sketch: KLLDynamic| { sketch.rebuild_capacity_cache(); sketch - }) - } + }), _ => Err(rmp_serde::decode::Error::Uncategorized(format!( - "KLLDynamic magic-ID mismatch: expected 0x{:02x}, got {:?}", + "KLLDynamic kind_id mismatch: expected [0x{:02x}, hasher], got {:?}", magic_ids::NATIVE_KLL_DYNAMIC, - bytes - .first() - .map(|b| format!("0x{b:02x}")) - .unwrap_or_else(|| "empty buffer".to_string()) + kind_id ))), } } diff --git a/src/sketches/kmv.rs b/src/sketches/kmv.rs index 6f5141f..6893fca 100644 --- a/src/sketches/kmv.rs +++ b/src/sketches/kmv.rs @@ -77,27 +77,28 @@ impl KMV { } } - /// Serializes the sketch into MessagePack bytes, prefixed with the - /// [`crate::message_pack_format::magic_ids::NATIVE_KMV`] magic byte. + /// Serializes the sketch into ASK1-wrapped MessagePack bytes. + /// kind_id: `[NATIVE_KMV, HASHER_UNKNOWN]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; - let mut out = vec![magic_ids::NATIVE_KMV, magic_ids::HASHER_UNKNOWN]; - out.extend(to_vec_named(self)?); - Ok(out) + let payload = to_vec_named(self)?; + Ok(magic_ids::encode_wrapper( + &[magic_ids::NATIVE_KMV, magic_ids::HASHER_UNKNOWN], + &payload, + )) } /// Deserializes a sketch from MessagePack bytes produced by [`Self::serialize_to_bytes`]. pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { use crate::message_pack_format::magic_ids; - match bytes { - [id, _hasher, rest @ ..] if *id == magic_ids::NATIVE_KMV => from_slice(rest), + let (kind_id, payload) = + magic_ids::decode_wrapper(bytes).map_err(RmpDecodeError::Uncategorized)?; + match kind_id { + [id, _hasher] if *id == magic_ids::NATIVE_KMV => from_slice(payload), _ => Err(RmpDecodeError::Uncategorized(format!( - "KMV magic-ID mismatch: expected 0x{:02x}, got {:?}", + "KMV kind_id mismatch: expected [0x{:02x}, hasher], got {:?}", magic_ids::NATIVE_KMV, - bytes - .first() - .map(|b| format!("0x{b:02x}")) - .unwrap_or_else(|| "empty buffer".to_string()) + kind_id ))), } } From b8f7de213b6259a51865bb633ef2e9d36163a11d Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Tue, 30 Jun 2026 00:28:15 -0400 Subject: [PATCH 08/11] fix(magic-id): correct hasher encoding, error paths, and doc typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - kmv.rs: encode H::hasher_magic_id() instead of HASHER_UNKNOWN; verify stored hasher on deserialize via check_hasher_id:: - hll.rs: validate stored hasher byte in HyperLogLogHIPImpl::deserialize_from_bytes (accept HASHER_DEFAULT_XX or HASHER_UNKNOWN, reject anything else) - portable/*/from_msgpack: propagate decode_wrapper structural failures as Error::Decode rather than silently swallowing them into a misleading BadMagicId { got: bytes[0] } (bytes[0] is always 0x41 'A' for ASK1 input) - docs/msgpack-magic-ids.md: fix copy-paste error — KLL inner bytes carry prefix 0x8a (NATIVE_KLL), not 0x87 (NATIVE_HLL_ERTL_MLE) Co-Authored-By: Claude Sonnet 4.6 (1M context) --- docs/msgpack-magic-ids.md | 2 +- src/message_pack_format/portable/countminsketch.rs | 5 ++--- .../portable/countminsketch_topk.rs | 5 ++--- src/message_pack_format/portable/countsketch.rs | 5 ++--- src/message_pack_format/portable/ddsketch.rs | 5 ++--- .../portable/delta_set_aggregator.rs | 5 ++--- src/message_pack_format/portable/hll.rs | 5 ++--- src/message_pack_format/portable/hydra_kll.rs | 5 ++--- src/message_pack_format/portable/kll.rs | 5 ++--- src/message_pack_format/portable/set_aggregator.rs | 5 ++--- src/sketches/hll.rs | 11 ++++++++++- src/sketches/kmv.rs | 9 ++++++--- 12 files changed, 35 insertions(+), 32 deletions(-) diff --git a/docs/msgpack-magic-ids.md b/docs/msgpack-magic-ids.md index 524b00d..625ccf2 100644 --- a/docs/msgpack-magic-ids.md +++ b/docs/msgpack-magic-ids.md @@ -95,7 +95,7 @@ encoding. The two are not interchangeable even for logically equivalent types `portable::KllSketch::to_msgpack` (`0x06`) embeds the raw KLL cell bytes inside a `KllSketchData { k, sketch_bytes }` msgpack struct. Those embedded bytes are produced by `KLL::serialize_to_bytes` and therefore carry the native -`0x87` prefix. The portable round-trip is: +`0x8a` prefix. The portable round-trip is: ``` KllSketch::to_msgpack() → [ ASK1 | v=1 | len=1 | 0x06 | msgpack([k, [ASK1|v=1|len=2|0x8a|0xff|raw_kll]]) ] diff --git a/src/message_pack_format/portable/countminsketch.rs b/src/message_pack_format/portable/countminsketch.rs index 52b8bbf..3ee47e2 100644 --- a/src/message_pack_format/portable/countminsketch.rs +++ b/src/message_pack_format/portable/countminsketch.rs @@ -451,9 +451,8 @@ impl MessagePackCodec for CountMinSketch { fn from_msgpack(bytes: &[u8]) -> Result { let (kind_id, payload) = - magic_ids::decode_wrapper(bytes).map_err(|_| MsgPackError::BadMagicId { - expected: magic_ids::COUNT_MIN_SKETCH, - got: bytes.first().copied(), + magic_ids::decode_wrapper(bytes).map_err(|msg| { + MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)) })?; if kind_id != [magic_ids::COUNT_MIN_SKETCH] { return Err(MsgPackError::BadMagicId { diff --git a/src/message_pack_format/portable/countminsketch_topk.rs b/src/message_pack_format/portable/countminsketch_topk.rs index c7d3ff0..7cf110b 100644 --- a/src/message_pack_format/portable/countminsketch_topk.rs +++ b/src/message_pack_format/portable/countminsketch_topk.rs @@ -306,9 +306,8 @@ impl MessagePackCodec for CountMinSketchWithHeap { fn from_msgpack(bytes: &[u8]) -> Result { let (kind_id, payload) = - magic_ids::decode_wrapper(bytes).map_err(|_| MsgPackError::BadMagicId { - expected: magic_ids::COUNT_MIN_SKETCH_WITH_HEAP, - got: bytes.first().copied(), + magic_ids::decode_wrapper(bytes).map_err(|msg| { + MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)) })?; if kind_id != [magic_ids::COUNT_MIN_SKETCH_WITH_HEAP] { return Err(MsgPackError::BadMagicId { diff --git a/src/message_pack_format/portable/countsketch.rs b/src/message_pack_format/portable/countsketch.rs index fb20d96..12c0fcf 100644 --- a/src/message_pack_format/portable/countsketch.rs +++ b/src/message_pack_format/portable/countsketch.rs @@ -443,9 +443,8 @@ impl MessagePackCodec for CountSketch { fn from_msgpack(bytes: &[u8]) -> Result { let (kind_id, payload) = - magic_ids::decode_wrapper(bytes).map_err(|_| MsgPackError::BadMagicId { - expected: magic_ids::COUNT_SKETCH, - got: bytes.first().copied(), + magic_ids::decode_wrapper(bytes).map_err(|msg| { + MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)) })?; if kind_id != [magic_ids::COUNT_SKETCH] { return Err(MsgPackError::BadMagicId { diff --git a/src/message_pack_format/portable/ddsketch.rs b/src/message_pack_format/portable/ddsketch.rs index 71aa67b..182f9e4 100644 --- a/src/message_pack_format/portable/ddsketch.rs +++ b/src/message_pack_format/portable/ddsketch.rs @@ -409,9 +409,8 @@ impl MessagePackCodec for DdSketch { fn from_msgpack(bytes: &[u8]) -> Result { let (kind_id, payload) = - magic_ids::decode_wrapper(bytes).map_err(|_| MsgPackError::BadMagicId { - expected: magic_ids::DD_SKETCH, - got: bytes.first().copied(), + magic_ids::decode_wrapper(bytes).map_err(|msg| { + MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)) })?; if kind_id != [magic_ids::DD_SKETCH] { return Err(MsgPackError::BadMagicId { diff --git a/src/message_pack_format/portable/delta_set_aggregator.rs b/src/message_pack_format/portable/delta_set_aggregator.rs index 635dedf..3d1c5da 100644 --- a/src/message_pack_format/portable/delta_set_aggregator.rs +++ b/src/message_pack_format/portable/delta_set_aggregator.rs @@ -29,9 +29,8 @@ impl MessagePackCodec for DeltaResult { fn from_msgpack(bytes: &[u8]) -> Result { let (kind_id, payload) = - magic_ids::decode_wrapper(bytes).map_err(|_| MsgPackError::BadMagicId { - expected: magic_ids::DELTA_RESULT, - got: bytes.first().copied(), + magic_ids::decode_wrapper(bytes).map_err(|msg| { + MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)) })?; if kind_id != [magic_ids::DELTA_RESULT] { return Err(MsgPackError::BadMagicId { diff --git a/src/message_pack_format/portable/hll.rs b/src/message_pack_format/portable/hll.rs index 5bdf722..5cb9089 100644 --- a/src/message_pack_format/portable/hll.rs +++ b/src/message_pack_format/portable/hll.rs @@ -436,9 +436,8 @@ impl MessagePackCodec for HllSketch { fn from_msgpack(bytes: &[u8]) -> Result { let (kind_id, payload) = - magic_ids::decode_wrapper(bytes).map_err(|_| MsgPackError::BadMagicId { - expected: magic_ids::HLL, - got: bytes.first().copied(), + magic_ids::decode_wrapper(bytes).map_err(|msg| { + MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)) })?; if kind_id != [magic_ids::HLL] { return Err(MsgPackError::BadMagicId { diff --git a/src/message_pack_format/portable/hydra_kll.rs b/src/message_pack_format/portable/hydra_kll.rs index 5b4a802..96d741a 100644 --- a/src/message_pack_format/portable/hydra_kll.rs +++ b/src/message_pack_format/portable/hydra_kll.rs @@ -169,9 +169,8 @@ impl MessagePackCodec for HydraKllSketch { use rmp_serde::decode::Error as RmpDecodeError; let (kind_id, payload) = - magic_ids::decode_wrapper(bytes).map_err(|_| MsgPackError::BadMagicId { - expected: magic_ids::HYDRA_KLL_SKETCH, - got: bytes.first().copied(), + magic_ids::decode_wrapper(bytes).map_err(|msg| { + MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)) })?; if kind_id != [magic_ids::HYDRA_KLL_SKETCH] { return Err(MsgPackError::BadMagicId { diff --git a/src/message_pack_format/portable/kll.rs b/src/message_pack_format/portable/kll.rs index c065ea2..a12a484 100644 --- a/src/message_pack_format/portable/kll.rs +++ b/src/message_pack_format/portable/kll.rs @@ -356,9 +356,8 @@ impl MessagePackCodec for KllSketch { fn from_msgpack(bytes: &[u8]) -> Result { let (kind_id, payload) = - magic_ids::decode_wrapper(bytes).map_err(|_| MsgPackError::BadMagicId { - expected: magic_ids::KLL_SKETCH, - got: bytes.first().copied(), + magic_ids::decode_wrapper(bytes).map_err(|msg| { + MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)) })?; if kind_id != [magic_ids::KLL_SKETCH] { return Err(MsgPackError::BadMagicId { diff --git a/src/message_pack_format/portable/set_aggregator.rs b/src/message_pack_format/portable/set_aggregator.rs index 7ae4ac4..cd69bec 100644 --- a/src/message_pack_format/portable/set_aggregator.rs +++ b/src/message_pack_format/portable/set_aggregator.rs @@ -90,9 +90,8 @@ impl MessagePackCodec for SetAggregator { fn from_msgpack(bytes: &[u8]) -> Result { let (kind_id, payload) = - magic_ids::decode_wrapper(bytes).map_err(|_| MsgPackError::BadMagicId { - expected: magic_ids::SET_AGGREGATOR, - got: bytes.first().copied(), + magic_ids::decode_wrapper(bytes).map_err(|msg| { + MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)) })?; if kind_id != [magic_ids::SET_AGGREGATOR] { return Err(MsgPackError::BadMagicId { diff --git a/src/sketches/hll.rs b/src/sketches/hll.rs index 7dab5ef..c18b3ce 100644 --- a/src/sketches/hll.rs +++ b/src/sketches/hll.rs @@ -453,7 +453,16 @@ impl HyperLogLogHIPImpl { let (kind_id, payload) = magic_ids::decode_wrapper(bytes).map_err(RmpDecodeError::Uncategorized)?; match kind_id { - [id, _hasher] if *id == magic_ids::NATIVE_HLL_HIP => from_slice(payload), + [id, hasher] if *id == magic_ids::NATIVE_HLL_HIP => { + if *hasher != magic_ids::HASHER_DEFAULT_XX && *hasher != magic_ids::HASHER_UNKNOWN { + return Err(RmpDecodeError::Uncategorized(format!( + "HyperLogLogHIPImpl hasher mismatch: expected 0x{:02x}, got 0x{:02x}", + magic_ids::HASHER_DEFAULT_XX, + hasher + ))); + } + from_slice(payload) + } _ => Err(RmpDecodeError::Uncategorized(format!( "HyperLogLogHIPImpl kind_id mismatch: expected [0x{:02x}, hasher], got {:?}", magic_ids::NATIVE_HLL_HIP, diff --git a/src/sketches/kmv.rs b/src/sketches/kmv.rs index 6893fca..13f5e7a 100644 --- a/src/sketches/kmv.rs +++ b/src/sketches/kmv.rs @@ -78,12 +78,12 @@ impl KMV { } /// Serializes the sketch into ASK1-wrapped MessagePack bytes. - /// kind_id: `[NATIVE_KMV, HASHER_UNKNOWN]`. + /// kind_id: `[NATIVE_KMV, H::hasher_magic_id()]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; let payload = to_vec_named(self)?; Ok(magic_ids::encode_wrapper( - &[magic_ids::NATIVE_KMV, magic_ids::HASHER_UNKNOWN], + &[magic_ids::NATIVE_KMV, H::hasher_magic_id()], &payload, )) } @@ -94,7 +94,10 @@ impl KMV { let (kind_id, payload) = magic_ids::decode_wrapper(bytes).map_err(RmpDecodeError::Uncategorized)?; match kind_id { - [id, _hasher] if *id == magic_ids::NATIVE_KMV => from_slice(payload), + [id, hasher] if *id == magic_ids::NATIVE_KMV => { + magic_ids::check_hasher_id::(*hasher)?; + from_slice(payload) + } _ => Err(RmpDecodeError::Uncategorized(format!( "KMV kind_id mismatch: expected [0x{:02x}, hasher], got {:?}", magic_ids::NATIVE_KMV, From e0f597e8d29be20e293d561e3019984a311d7b67 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Tue, 30 Jun 2026 00:47:02 -0400 Subject: [PATCH 09/11] chore: cargo fmt + clippy fixes Auto-fixed uninlined format args (clippy::uninlined_format_args) and applied cargo fmt across the tree. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- src/common/input.rs | 2 +- src/common/structure_utils.rs | 9 +- .../portable/countminsketch.rs | 6 +- .../portable/countminsketch_topk.rs | 8 +- .../portable/countsketch.rs | 6 +- src/message_pack_format/portable/ddsketch.rs | 19 ++-- .../portable/delta_set_aggregator.rs | 6 +- src/message_pack_format/portable/hll.rs | 6 +- src/message_pack_format/portable/hydra_kll.rs | 6 +- src/message_pack_format/portable/kll.rs | 6 +- .../portable/set_aggregator.rs | 6 +- src/sketch_framework/hydra.rs | 12 +-- src/sketch_framework/univmon.rs | 4 +- src/sketches/ddsketch.rs | 32 +------ src/sketches/kll.rs | 6 +- src/sketches/kll_dynamic.rs | 6 +- tests/xtest_consumer.rs | 88 ++++++------------- 17 files changed, 71 insertions(+), 157 deletions(-) diff --git a/src/common/input.rs b/src/common/input.rs index 0fbd337..2eb3e19 100644 --- a/src/common/input.rs +++ b/src/common/input.rs @@ -483,7 +483,7 @@ impl HydraCounter { (HydraCounter::UNIVERSAL(um), HydraQuery::L1Norm) => Ok(um.calc_l1()), (HydraCounter::UNIVERSAL(um), HydraQuery::L2Norm) => Ok(um.calc_l2()), (HydraCounter::UNIVERSAL(um), HydraQuery::Entropy) => Ok(um.calc_entropy()), - (c, q) => Err(format!("{} does not support {}", c, q)), + (c, q) => Err(format!("{c} does not support {q}")), } } diff --git a/src/common/structure_utils.rs b/src/common/structure_utils.rs index 29e9165..a17e461 100644 --- a/src/common/structure_utils.rs +++ b/src/common/structure_utils.rs @@ -365,8 +365,7 @@ mod tests { let sort_median = median_three_sort(v); assert_eq!( fast_median, sort_median, - "median for sort is {sort_median} but fast gives {fast_median}, input is {:?}", - v + "median for sort is {sort_median} but fast gives {fast_median}, input is {v:?}" ); } for v in &mut four_vec { @@ -374,8 +373,7 @@ mod tests { let sort_median = median_four_sort(v); assert_eq!( fast_median, sort_median, - "median for sort is {sort_median} but fast gives {fast_median}, input is {:?}", - v + "median for sort is {sort_median} but fast gives {fast_median}, input is {v:?}" ); } for v in &mut five_vec { @@ -383,8 +381,7 @@ mod tests { let sort_median = median_five_sort(v); assert_eq!( fast_median, sort_median, - "median for sort is {sort_median} but fast gives {fast_median}, input is {:?}", - v + "median for sort is {sort_median} but fast gives {fast_median}, input is {v:?}" ); } } diff --git a/src/message_pack_format/portable/countminsketch.rs b/src/message_pack_format/portable/countminsketch.rs index 3ee47e2..b99264c 100644 --- a/src/message_pack_format/portable/countminsketch.rs +++ b/src/message_pack_format/portable/countminsketch.rs @@ -450,10 +450,8 @@ impl MessagePackCodec for CountMinSketch { } fn from_msgpack(bytes: &[u8]) -> Result { - let (kind_id, payload) = - magic_ids::decode_wrapper(bytes).map_err(|msg| { - MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)) - })?; + let (kind_id, payload) = magic_ids::decode_wrapper(bytes) + .map_err(|msg| MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)))?; if kind_id != [magic_ids::COUNT_MIN_SKETCH] { return Err(MsgPackError::BadMagicId { expected: magic_ids::COUNT_MIN_SKETCH, diff --git a/src/message_pack_format/portable/countminsketch_topk.rs b/src/message_pack_format/portable/countminsketch_topk.rs index 7cf110b..35d962b 100644 --- a/src/message_pack_format/portable/countminsketch_topk.rs +++ b/src/message_pack_format/portable/countminsketch_topk.rs @@ -86,7 +86,7 @@ pub fn heap_to_wire(cms_heap: &SketchlibCMSHeap) -> Vec { .map(|hh_item| { let key = match &hh_item.key { crate::HeapItem::String(s) => s.clone(), - other => format!("{:?}", other), + other => format!("{other:?}"), }; WireHeapItem { key, @@ -305,10 +305,8 @@ impl MessagePackCodec for CountMinSketchWithHeap { } fn from_msgpack(bytes: &[u8]) -> Result { - let (kind_id, payload) = - magic_ids::decode_wrapper(bytes).map_err(|msg| { - MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)) - })?; + let (kind_id, payload) = magic_ids::decode_wrapper(bytes) + .map_err(|msg| MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)))?; if kind_id != [magic_ids::COUNT_MIN_SKETCH_WITH_HEAP] { return Err(MsgPackError::BadMagicId { expected: magic_ids::COUNT_MIN_SKETCH_WITH_HEAP, diff --git a/src/message_pack_format/portable/countsketch.rs b/src/message_pack_format/portable/countsketch.rs index 12c0fcf..2d8c3a7 100644 --- a/src/message_pack_format/portable/countsketch.rs +++ b/src/message_pack_format/portable/countsketch.rs @@ -442,10 +442,8 @@ impl MessagePackCodec for CountSketch { } fn from_msgpack(bytes: &[u8]) -> Result { - let (kind_id, payload) = - magic_ids::decode_wrapper(bytes).map_err(|msg| { - MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)) - })?; + let (kind_id, payload) = magic_ids::decode_wrapper(bytes) + .map_err(|msg| MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)))?; if kind_id != [magic_ids::COUNT_SKETCH] { return Err(MsgPackError::BadMagicId { expected: magic_ids::COUNT_SKETCH, diff --git a/src/message_pack_format/portable/ddsketch.rs b/src/message_pack_format/portable/ddsketch.rs index 182f9e4..7917ac6 100644 --- a/src/message_pack_format/portable/ddsketch.rs +++ b/src/message_pack_format/portable/ddsketch.rs @@ -408,10 +408,8 @@ impl MessagePackCodec for DdSketch { } fn from_msgpack(bytes: &[u8]) -> Result { - let (kind_id, payload) = - magic_ids::decode_wrapper(bytes).map_err(|msg| { - MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)) - })?; + let (kind_id, payload) = magic_ids::decode_wrapper(bytes) + .map_err(|msg| MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)))?; if kind_id != [magic_ids::DD_SKETCH] { return Err(MsgPackError::BadMagicId { expected: magic_ids::DD_SKETCH, @@ -602,13 +600,11 @@ mod tests { // P99 ≈ exp(mu + sigma * Φ⁻¹(0.99)) = e^(3 + 0.7×2.326) ≈ 102.4. assert!( (p50 / 20.09).ln().abs() < 0.05, - "P50 {} not close to 20.09", - p50 + "P50 {p50} not close to 20.09" ); assert!( (p99 / 102.4).ln().abs() < 0.05, - "P99 {} not close to 102.4", - p99 + "P99 {p99} not close to 102.4" ); } @@ -682,12 +678,7 @@ mod tests { let rel_err = (got / want - 1.0).abs(); assert!( rel_err <= alpha, - "q={} rel_err={:.4} exceeds α={}: reconstituted={}, full={}", - q, - rel_err, - alpha, - got, - want, + "q={q} rel_err={rel_err:.4} exceeds α={alpha}: reconstituted={got}, full={want}", ); } } diff --git a/src/message_pack_format/portable/delta_set_aggregator.rs b/src/message_pack_format/portable/delta_set_aggregator.rs index 3d1c5da..46228f7 100644 --- a/src/message_pack_format/portable/delta_set_aggregator.rs +++ b/src/message_pack_format/portable/delta_set_aggregator.rs @@ -28,10 +28,8 @@ impl MessagePackCodec for DeltaResult { } fn from_msgpack(bytes: &[u8]) -> Result { - let (kind_id, payload) = - magic_ids::decode_wrapper(bytes).map_err(|msg| { - MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)) - })?; + let (kind_id, payload) = magic_ids::decode_wrapper(bytes) + .map_err(|msg| MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)))?; if kind_id != [magic_ids::DELTA_RESULT] { return Err(MsgPackError::BadMagicId { expected: magic_ids::DELTA_RESULT, diff --git a/src/message_pack_format/portable/hll.rs b/src/message_pack_format/portable/hll.rs index 5cb9089..949a7e6 100644 --- a/src/message_pack_format/portable/hll.rs +++ b/src/message_pack_format/portable/hll.rs @@ -435,10 +435,8 @@ impl MessagePackCodec for HllSketch { } fn from_msgpack(bytes: &[u8]) -> Result { - let (kind_id, payload) = - magic_ids::decode_wrapper(bytes).map_err(|msg| { - MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)) - })?; + let (kind_id, payload) = magic_ids::decode_wrapper(bytes) + .map_err(|msg| MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)))?; if kind_id != [magic_ids::HLL] { return Err(MsgPackError::BadMagicId { expected: magic_ids::HLL, diff --git a/src/message_pack_format/portable/hydra_kll.rs b/src/message_pack_format/portable/hydra_kll.rs index 96d741a..ee612f9 100644 --- a/src/message_pack_format/portable/hydra_kll.rs +++ b/src/message_pack_format/portable/hydra_kll.rs @@ -168,10 +168,8 @@ impl MessagePackCodec for HydraKllSketch { use crate::sketches::kll::KLL; use rmp_serde::decode::Error as RmpDecodeError; - let (kind_id, payload) = - magic_ids::decode_wrapper(bytes).map_err(|msg| { - MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)) - })?; + let (kind_id, payload) = magic_ids::decode_wrapper(bytes) + .map_err(|msg| MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)))?; if kind_id != [magic_ids::HYDRA_KLL_SKETCH] { return Err(MsgPackError::BadMagicId { expected: magic_ids::HYDRA_KLL_SKETCH, diff --git a/src/message_pack_format/portable/kll.rs b/src/message_pack_format/portable/kll.rs index a12a484..959e732 100644 --- a/src/message_pack_format/portable/kll.rs +++ b/src/message_pack_format/portable/kll.rs @@ -355,10 +355,8 @@ impl MessagePackCodec for KllSketch { } fn from_msgpack(bytes: &[u8]) -> Result { - let (kind_id, payload) = - magic_ids::decode_wrapper(bytes).map_err(|msg| { - MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)) - })?; + let (kind_id, payload) = magic_ids::decode_wrapper(bytes) + .map_err(|msg| MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)))?; if kind_id != [magic_ids::KLL_SKETCH] { return Err(MsgPackError::BadMagicId { expected: magic_ids::KLL_SKETCH, diff --git a/src/message_pack_format/portable/set_aggregator.rs b/src/message_pack_format/portable/set_aggregator.rs index cd69bec..0c49c48 100644 --- a/src/message_pack_format/portable/set_aggregator.rs +++ b/src/message_pack_format/portable/set_aggregator.rs @@ -89,10 +89,8 @@ impl MessagePackCodec for SetAggregator { } fn from_msgpack(bytes: &[u8]) -> Result { - let (kind_id, payload) = - magic_ids::decode_wrapper(bytes).map_err(|msg| { - MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)) - })?; + let (kind_id, payload) = magic_ids::decode_wrapper(bytes) + .map_err(|msg| MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)))?; if kind_id != [magic_ids::SET_AGGREGATOR] { return Err(MsgPackError::BadMagicId { expected: magic_ids::SET_AGGREGATOR, diff --git a/src/sketch_framework/hydra.rs b/src/sketch_framework/hydra.rs index 8f9f627..c7db0a3 100644 --- a/src/sketch_framework/hydra.rs +++ b/src/sketch_framework/hydra.rs @@ -307,8 +307,7 @@ impl MultiHeadHydra { } if std::mem::discriminant(counter) != std::mem::discriminant(other_counter) { return Err(format!( - "MultiHeadHydra counter type mismatch for dimension '{}'", - name + "MultiHeadHydra counter type mismatch for dimension '{name}'" )); } } @@ -731,8 +730,7 @@ mod tests { let quantile = hydra.query_key(vec!["metrics", "latency"], &HydraQuery::Cdf(30.0)); assert!( (quantile - 0.6).abs() < 1e-9, - "expected CDF near 0.6, got {}", - quantile + "expected CDF near 0.6, got {quantile}" ); let empty_bucket = hydra.query_key(vec!["other", "key"], &HydraQuery::Cdf(50.0)); @@ -851,8 +849,7 @@ mod tests { let card = result.unwrap(); assert!( card > 90.0 && card < 110.0, - "Expected approx 100, got {}", - card + "Expected approx 100, got {card}" ); } @@ -874,8 +871,7 @@ mod tests { let median = result.unwrap(); assert!( (median - 50.0).abs() < 5.0, - "Expected approx 50, got {}", - median + "Expected approx 50, got {median}" ); } diff --git a/src/sketch_framework/univmon.rs b/src/sketch_framework/univmon.rs index b7caf59..b08dd76 100644 --- a/src/sketch_framework/univmon.rs +++ b/src/sketch_framework/univmon.rs @@ -146,7 +146,7 @@ impl UnivMon { pub fn print_hh_layer(&self) { print!("Print HH_Layer: "); for i in 0..self.layer_size { - println!("layer {}: ", i); + println!("layer {i}: "); self.hh_layers[i].print_heap(); } } @@ -651,7 +651,7 @@ mod tests { for (prefix, count, repeat) in scenarios { for i in 0..repeat { - let key = format!("{}_{}", prefix, i); + let key = format!("{prefix}_{i}"); let val = count as i64; let val_f = val as f64; diff --git a/src/sketches/ddsketch.rs b/src/sketches/ddsketch.rs index 4204139..4c938e1 100644 --- a/src/sketches/ddsketch.rs +++ b/src/sketches/ddsketch.rs @@ -542,13 +542,7 @@ mod tests { let err = rel_err(got, want); assert!( err <= tol, - "quantile {} (p={:.2}) relerr={:.4} got={} want={} tol={}", - name, - p, - err, - got, - want, - tol + "quantile {name} (p={p:.2}) relerr={err:.4} got={got} want={want} tol={tol}" ); } } @@ -609,13 +603,7 @@ mod tests { let err = rel_err(got, want); assert!( err <= tol, - "quantile {} (p={:.2}) relerr={:.4} got={} want={} tol={}", - name, - p, - err, - got, - want, - tol + "quantile {name} (p={p:.2}) relerr={err:.4} got={got} want={want} tol={tol}" ); } } @@ -679,13 +667,7 @@ mod tests { let err = rel_err(got, want); assert!( err <= tol, - "quantile {} (p={:.2}) relerr={:.4} got={} want={} tol={}", - name, - p, - err, - got, - want, - tol + "quantile {name} (p={p:.2}) relerr={err:.4} got={got} want={want} tol={tol}" ); } } @@ -746,13 +728,7 @@ mod tests { let err = rel_err(got, want); assert!( err <= tol + 1e-9, - "quantile {} (p={:.2}) relerr={:.4} got={} want={} tol={}", - name, - p, - err, - got, - want, - tol + "quantile {name} (p={p:.2}) relerr={err:.4} got={got} want={want} tol={tol}" ); } } diff --git a/src/sketches/kll.rs b/src/sketches/kll.rs index e65a042..b7b7203 100644 --- a/src/sketches/kll.rs +++ b/src/sketches/kll.rs @@ -1188,7 +1188,7 @@ mod tests { let median = cdf.query(0.5); // Median should be 30.5 - assert!(median > 20.0 && median < 40.2, "Median = {}", median); + assert!(median > 20.0 && median < 40.2, "Median = {median}"); // Test error handling for non-numeric input let result = kll.update_data_input(&DataInput::String("not a number".to_string())); @@ -1218,7 +1218,7 @@ mod tests { // cdf.print_entries(); let median = cdf.query(0.5); // only 30 and 40 is possible - assert!(median == 30.0 || median == 40.0, "Median = {}", median); + assert!(median == 30.0 || median == 40.0, "Median = {median}"); } #[test] @@ -1243,7 +1243,7 @@ mod tests { // kll.print_compactors(); let median = cdf.query(0.5); // Median should be 30 - assert!(median == 30.0, "Median = {}", median); + assert!(median == 30.0, "Median = {median}"); } #[test] diff --git a/src/sketches/kll_dynamic.rs b/src/sketches/kll_dynamic.rs index 51759e3..5f381aa 100644 --- a/src/sketches/kll_dynamic.rs +++ b/src/sketches/kll_dynamic.rs @@ -646,7 +646,7 @@ mod tests { let median = cdf.query(0.5); // Median should be 30.5 - assert!(median > 20.0 && median < 40.2, "Median = {}", median); + assert!(median > 20.0 && median < 40.2, "Median = {median}"); // Test error handling for non-numeric input let result = kll.update_data_input(&DataInput::String("not a number".to_string())); @@ -669,7 +669,7 @@ mod tests { let cdf = kll.cdf(); let median = cdf.query(0.5); // only 30 and 40 is possible - assert!(median == 30.0 || median == 40.0, "Median = {}", median); + assert!(median == 30.0 || median == 40.0, "Median = {median}"); } #[test] @@ -686,7 +686,7 @@ mod tests { let cdf = kll.cdf(); let median = cdf.query(0.5); // Median should be 30 - assert!(median == 30.0, "Median = {}", median); + assert!(median == 30.0, "Median = {median}"); } #[test] diff --git a/tests/xtest_consumer.rs b/tests/xtest_consumer.rs index ce9364e..ea9332a 100644 --- a/tests/xtest_consumer.rs +++ b/tests/xtest_consumer.rs @@ -62,7 +62,7 @@ fn cross_language_proto() { let cm_state = match env.sketch_state { Some(sketch_envelope::SketchState::CountMin(ref s)) => s.clone(), - other => panic!("expected CountMin sketch_state, got {:?}", other), + other => panic!("expected CountMin sketch_state, got {other:?}"), }; println!( @@ -96,10 +96,7 @@ fn cross_language_proto() { let hot_key = b"item:42" as &[u8]; let hot_hash = xxh3_64_seeded(SEED_0, hot_key); - println!( - "[CountMin] Step 3/3 — Point query 'item:42' (hash=0x{:016x})", - hot_hash - ); + println!("[CountMin] Step 3/3 — Point query 'item:42' (hash=0x{hot_hash:016x})"); let bits_per_row = col_bits(cols); let mask = (cols as u64) - 1; @@ -112,12 +109,9 @@ fn cross_language_proto() { min_freq = v; } } - println!( - "[CountMin] min frequency estimate = {:.0} (expect ≥ 101)", - min_freq - ); + println!("[CountMin] min frequency estimate = {min_freq:.0} (expect ≥ 101)"); if min_freq < 101.0 { - eprintln!("[CountMin] FAIL: frequency {:.0} < 101", min_freq); + eprintln!("[CountMin] FAIL: frequency {min_freq:.0} < 101"); all_ok = false; } else { println!("[CountMin] PASS"); @@ -139,7 +133,7 @@ fn cross_language_proto() { let kll_state = match env.sketch_state { Some(sketch_envelope::SketchState::Kll(ref s)) => s.clone(), - other => panic!("expected KLL sketch_state, got {:?}", other), + other => panic!("expected KLL sketch_state, got {other:?}"), }; println!( @@ -154,17 +148,14 @@ fn cross_language_proto() { let kll = KllFromProto::from_state(&kll_state); let p50 = kll.quantile(0.50); let p99 = kll.quantile(0.99); - println!( - "[KLL] Step 3/3 — p50 ≈ {:.1} (expect ~5000) p99 ≈ {:.1} (expect ~9900)", - p50, p99 - ); + println!("[KLL] Step 3/3 — p50 ≈ {p50:.1} (expect ~5000) p99 ≈ {p99:.1} (expect ~9900)"); let p50_ok = (p50 - 5000.0).abs() / 5000.0 < 0.05; let p99_ok = (p99 - 9900.0).abs() / 9900.0 < 0.05; if p50_ok && p99_ok { println!("[KLL] PASS"); } else { - eprintln!("[KLL] FAIL: p50={:.1} p99={:.1}", p50, p99); + eprintln!("[KLL] FAIL: p50={p50:.1} p99={p99:.1}"); all_ok = false; } @@ -184,7 +175,7 @@ fn cross_language_proto() { let dd_state = match env.sketch_state { Some(sketch_envelope::SketchState::Ddsketch(ref s)) => s.clone(), - other => panic!("expected DDSketch sketch_state, got {:?}", other), + other => panic!("expected DDSketch sketch_state, got {other:?}"), }; // `count` was dropped from the wire (ProjectASAP/sketchlib-go#243); @@ -216,7 +207,7 @@ fn cross_language_proto() { if p50_ok && p99_ok { println!("[DDSketch] PASS"); } else { - eprintln!("[DDSketch] FAIL: p50={:?} p99={:?}", p50_dd, p99_dd); + eprintln!("[DDSketch] FAIL: p50={p50_dd:?} p99={p99_dd:?}"); all_ok = false; } @@ -236,7 +227,7 @@ fn cross_language_proto() { let hll_state = match env.sketch_state { Some(sketch_envelope::SketchState::Hll(ref s)) => s.clone(), - other => panic!("expected HLL sketch_state, got {:?}", other), + other => panic!("expected HLL sketch_state, got {other:?}"), }; println!( @@ -247,14 +238,11 @@ fn cross_language_proto() { ); let hll_card = hll_ertl_mle_estimate(&hll_state); - println!( - "[HLL] Step 3/3 — cardinality ≈ {} (expect ~50000)", - hll_card - ); + println!("[HLL] Step 3/3 — cardinality ≈ {hll_card} (expect ~50000)"); if (40_000..=65_000).contains(&hll_card) { println!("[HLL] PASS"); } else { - eprintln!("[HLL] FAIL: cardinality {} not in [40000, 65000]", hll_card); + eprintln!("[HLL] FAIL: cardinality {hll_card} not in [40000, 65000]"); all_ok = false; } @@ -274,7 +262,7 @@ fn cross_language_proto() { let cs_state = match env.sketch_state { Some(sketch_envelope::SketchState::CountSketch(ref s)) => s.clone(), - other => panic!("expected CountSketch sketch_state, got {:?}", other), + other => panic!("expected CountSketch sketch_state, got {other:?}"), }; println!( @@ -288,14 +276,11 @@ fn cross_language_proto() { // Go hash: common.Hash64([]byte("cs:hot")) = xxh3_64_seeded(seedList[0], b"cs:hot") let cs_hot_hash = xxh3_64_seeded(SEED_0, b"cs:hot"); let cs_est = count_sketch_query_float(&cs_state, cs_hot_hash); - println!( - "[CountSketch] Step 3/3 — 'cs:hot' est = {:.0} (expect ≥ 200)", - cs_est - ); + println!("[CountSketch] Step 3/3 — 'cs:hot' est = {cs_est:.0} (expect ≥ 200)"); if cs_est >= 200.0 { println!("[CountSketch] PASS"); } else { - eprintln!("[CountSketch] FAIL: estimate {:.0} < 200", cs_est); + eprintln!("[CountSketch] FAIL: estimate {cs_est:.0} < 200"); all_ok = false; } @@ -315,7 +300,7 @@ fn cross_language_proto() { let coco_state = match env.sketch_state { Some(sketch_envelope::SketchState::Coco(ref s)) => s.clone(), - other => panic!("expected CocoSketch sketch_state, got {:?}", other), + other => panic!("expected CocoSketch sketch_state, got {other:?}"), }; println!( @@ -330,14 +315,11 @@ fn cross_language_proto() { // DeriveIndex(hash, row, width): col = (hash >> (row * maskBitsForWidth(width))) & (width-1) let coco_hash = xxh3_64_seeded(SEED_0, b"coco:hot"); let coco_est = coco_estimate(&coco_state, coco_hash); - println!( - "[CocoSketch] Step 3/3 — 'coco:hot' est = {} (expect ≥ 500)", - coco_est - ); + println!("[CocoSketch] Step 3/3 — 'coco:hot' est = {coco_est} (expect ≥ 500)"); if coco_est >= 500 { println!("[CocoSketch] PASS"); } else { - eprintln!("[CocoSketch] FAIL: estimate {} < 500", coco_est); + eprintln!("[CocoSketch] FAIL: estimate {coco_est} < 500"); all_ok = false; } @@ -357,7 +339,7 @@ fn cross_language_proto() { let elastic_state = match env.sketch_state { Some(sketch_envelope::SketchState::Elastic(ref s)) => s.clone(), - other => panic!("expected ElasticSketch sketch_state, got {:?}", other), + other => panic!("expected ElasticSketch sketch_state, got {other:?}"), }; println!( @@ -371,14 +353,11 @@ fn cross_language_proto() { // Go uses CanonicalHashSeed = seedList[5] = 0x6a09e667 let elephant_hash = xxh3_64_seeded(SEED_5, b"elephant"); let elephant_est = elastic_query(&elastic_state, "elephant", elephant_hash); - println!( - "[ElasticSketch] Step 3/3 — 'elephant' est = {} (expect ≥ 900)", - elephant_est - ); + println!("[ElasticSketch] Step 3/3 — 'elephant' est = {elephant_est} (expect ≥ 900)"); if elephant_est >= 900 { println!("[ElasticSketch] PASS"); } else { - eprintln!("[ElasticSketch] FAIL: estimate {} < 900", elephant_est); + eprintln!("[ElasticSketch] FAIL: estimate {elephant_est} < 900"); all_ok = false; } @@ -398,7 +377,7 @@ fn cross_language_proto() { let um_state = match env.sketch_state { Some(sketch_envelope::SketchState::Univmon(ref s)) => s.clone(), - other => panic!("expected UnivMon sketch_state, got {:?}", other), + other => panic!("expected UnivMon sketch_state, got {other:?}"), }; println!( @@ -409,17 +388,11 @@ fn cross_language_proto() { let um_card = univmon_cardinality(&um_state); // Note: the g-sum heuristic typically underestimates (Go itself reports ~4250 for 10k inserts). // We verify the Rust result matches Go's algorithm rather than the true cardinality. - println!( - "[UnivMon] Step 3/3 — cardinality ≈ {:.0} (g-sum heuristic, Go also ~4250)", - um_card - ); + println!("[UnivMon] Step 3/3 — cardinality ≈ {um_card:.0} (g-sum heuristic, Go also ~4250)"); if (1_000.0..=15_000.0).contains(&um_card) { println!("[UnivMon] PASS"); } else { - eprintln!( - "[UnivMon] FAIL: cardinality {:.0} not in [1000, 15000]", - um_card - ); + eprintln!("[UnivMon] FAIL: cardinality {um_card:.0} not in [1000, 15000]"); all_ok = false; } @@ -439,7 +412,7 @@ fn cross_language_proto() { let hydra_state = match env.sketch_state { Some(sketch_envelope::SketchState::Hydra(ref s)) => s.clone(), - other => panic!("expected Hydra sketch_state, got {:?}", other), + other => panic!("expected Hydra sketch_state, got {other:?}"), }; println!( @@ -456,14 +429,11 @@ fn cross_language_proto() { let hydra_subkey_hash = xxh3_64_seeded(SEED_6, b"hydra:42"); let hydra_value_hash = xxh3_64_seeded(SEED_0, b"hydra:42"); let hydra_est = hydra_query_cm(&hydra_state, hydra_subkey_hash, hydra_value_hash); - println!( - "[Hydra] Step 3/3 — 'hydra:42' est = {:.0} (expect ≥ 51)", - hydra_est - ); + println!("[Hydra] Step 3/3 — 'hydra:42' est = {hydra_est:.0} (expect ≥ 51)"); if hydra_est >= 51.0 { println!("[Hydra] PASS"); } else { - eprintln!("[Hydra] FAIL: estimate {:.0} < 51", hydra_est); + eprintln!("[Hydra] FAIL: estimate {hydra_est:.0} < 51"); all_ok = false; } @@ -488,7 +458,7 @@ fn cross_language_proto() { let cm_state = match env.sketch_state { Some(sketch_envelope::SketchState::CountMin(ref s)) => s.clone(), - other => panic!("expected CountMin sketch_state, got {:?}", other), + other => panic!("expected CountMin sketch_state, got {other:?}"), }; let rows = cm_state.rows as usize; let cols = cm_state.cols as usize; @@ -541,7 +511,7 @@ fn cross_language_proto() { let mut hll_state = match env.sketch_state { Some(sketch_envelope::SketchState::Hll(ref s)) => s.clone(), - other => panic!("expected HLL sketch_state, got {:?}", other), + other => panic!("expected HLL sketch_state, got {other:?}"), }; // Dual-read the registers (sampling thins them → may be sparse-encoded). let regs = From 1aba9f6a4b87c0fa332cfcef2d64a6b3f919c74b Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Tue, 7 Jul 2026 13:25:41 -0400 Subject: [PATCH 10/11] fix(msgpack): rename wrapper magic to ASAPv1 --- docs/msgpack-magic-ids.md | 14 ++--- src/message_pack_format/magic_ids.rs | 52 +++++++++++-------- src/message_pack_format/portable/ddsketch.rs | 6 +-- .../portable/set_aggregator.rs | 2 +- src/sketch_framework/hydra.rs | 2 +- src/sketch_framework/univmon.rs | 2 +- src/sketches/countminsketch.rs | 6 +-- src/sketches/countsketch.rs | 6 +-- src/sketches/countsketch_topk.rs | 2 +- src/sketches/ddsketch.rs | 2 +- src/sketches/hll.rs | 11 ++-- src/sketches/kll.rs | 2 +- src/sketches/kll_dynamic.rs | 2 +- src/sketches/kmv.rs | 2 +- 14 files changed, 60 insertions(+), 51 deletions(-) diff --git a/docs/msgpack-magic-ids.md b/docs/msgpack-magic-ids.md index 625ccf2..2eec6ba 100644 --- a/docs/msgpack-magic-ids.md +++ b/docs/msgpack-magic-ids.md @@ -1,18 +1,18 @@ -# ASK1 Sketch Wire Format +# ASAPv1 Sketch Wire Format Every serialised sketch binary produced by this library is wrapped in the -**ASK1 envelope**, a self-describing header that carries the sketch's +**ASAPv1 envelope**, a self-describing header that carries the sketch's type discriminant without a fixed-size ceiling on the number of types. ``` -┌────────────┬────────────┬──────────────────┬───────────────────────┬──────────────────────┐ -│ b"ASK1": 4B │ version: u8 │ kind_id_len: u8 │ kind_id: [kind_id_len] │ msgpack payload … │ -└────────────┴────────────┴──────────────────┴───────────────────────┴──────────────────────┘ +┌───────────────┬────────────┬──────────────────┬───────────────────────┬──────────────────────┐ +│ b"ASAPv1": 6B │ version: u8 │ kind_id_len: u8 │ kind_id: [kind_id_len] │ msgpack payload … │ +└───────────────┴────────────┴──────────────────┴───────────────────────┴──────────────────────┘ ``` | Field | Value | Notes | |-------|-------|-------| -| `b"ASK1"` | `0x41 0x53 0x4B 0x31` | 4-byte ASCII sentinel, not a valid msgpack prefix | +| `b"ASAPv1"` | `0x41 0x53 0x41 0x50 0x76 0x31` | 6-byte ASCII sentinel, not a valid msgpack prefix | | `version` | `0x01` | Increment only if the envelope layout changes | | `kind_id_len` | 1 or 2 | Number of `kind_id` bytes (1 for portable, 2 for native) | | `kind_id` | see tables below | Canonical big-endian, no leading zero bytes | @@ -98,7 +98,7 @@ bytes are produced by `KLL::serialize_to_bytes` and therefore carry the native `0x8a` prefix. The portable round-trip is: ``` -KllSketch::to_msgpack() → [ ASK1 | v=1 | len=1 | 0x06 | msgpack([k, [ASK1|v=1|len=2|0x8a|0xff|raw_kll]]) ] +KllSketch::to_msgpack() → [ ASAPv1 | v=1 | len=1 | 0x06 | msgpack([k, [ASAPv1|v=1|len=2|0x8a|0xff|raw_kll]]) ] KllSketch::from_msgpack() → decode_wrapper → kind_id=[0x06], payload=msgpack struct → decode struct: k + sketch_bytes → KLL::deserialize_from_bytes(sketch_bytes) diff --git a/src/message_pack_format/magic_ids.rs b/src/message_pack_format/magic_ids.rs index cd7a3b7..fa78768 100644 --- a/src/message_pack_format/magic_ids.rs +++ b/src/message_pack_format/magic_ids.rs @@ -1,14 +1,14 @@ //! Magic-ID constants and wrapper encoding for the ASAP sketch wire format. //! -//! Every serialized binary produced by this library is wrapped in the **ASK1 +//! Every serialized binary produced by this library is wrapped in the **ASAPv1 //! envelope**, a self-describing header that identifies the sketch type and //! reserves space for future metadata without a fixed-size ceiling: //! //! ```text -//! [ b"ASK1" | version: u8 | kind_id_len: u8 | kind_id: [u8; kind_id_len] | ] +//! [ b"ASAPv1" | version: u8 | kind_id_len: u8 | kind_id: [u8; kind_id_len] | ] //! ``` //! -//! * `b"ASK1"` — 4-byte ASCII sentinel; unambiguously not a msgpack value. +//! * `b"ASAPv1"` — 6-byte ASCII sentinel; unambiguously not a msgpack value. //! * `version` — envelope layout version; currently `0x01`. //! * `kind_id_len + kind_id` — variable-length sketch discriminant encoded as //! canonical unsigned big-endian with no leading zero bytes. For current @@ -28,18 +28,18 @@ // ── Envelope constants ──────────────────────────────────────────────────────── -/// 4-byte ASCII sentinel that opens every ASAP sketch binary. -pub const WRAPPER_MAGIC: &[u8; 4] = b"ASK1"; +/// 6-byte ASCII sentinel that opens every ASAP sketch binary. +pub const WRAPPER_MAGIC: &[u8; 6] = b"ASAPv1"; -/// Envelope layout version stored in byte 4. Increment if the header -/// structure (field order, field semantics) ever changes. +/// Envelope layout version stored immediately after `WRAPPER_MAGIC`. Increment +/// if the header structure (field order, field semantics) ever changes. pub const WRAPPER_VERSION: u8 = 0x01; -/// Prepend the ASK1 envelope to `payload` and return the complete binary. +/// Prepend the ASAPv1 envelope to `payload` and return the complete binary. /// /// `kind_id` identifies the sketch type (1–2 bytes for all current types). pub fn encode_wrapper(kind_id: &[u8], payload: &[u8]) -> Vec { - let mut out = Vec::with_capacity(4 + 1 + 1 + kind_id.len() + payload.len()); + let mut out = Vec::with_capacity(WRAPPER_MAGIC.len() + 1 + 1 + kind_id.len() + payload.len()); out.extend_from_slice(WRAPPER_MAGIC); out.push(WRAPPER_VERSION); out.push(kind_id.len() as u8); @@ -48,36 +48,44 @@ pub fn encode_wrapper(kind_id: &[u8], payload: &[u8]) -> Vec { out } -/// Strip the ASK1 envelope from `bytes` and return `(kind_id, payload)`. +/// Strip the ASAPv1 envelope from `bytes` and return `(kind_id, payload)`. /// /// Returns `Err(String)` on any structural mismatch so callers can convert to /// their own error type. pub fn decode_wrapper(bytes: &[u8]) -> Result<(&[u8], &[u8]), String> { - if bytes.len() < 7 { + let magic_len = WRAPPER_MAGIC.len(); + let version_offset = magic_len; + let kind_id_len_offset = magic_len + 1; + let kind_id_offset = magic_len + 2; + let min_len = kind_id_offset + 1; + + if bytes.len() < min_len { return Err(format!( - "ASK1 wrapper: too short ({} bytes, need ≥7)", + "ASAPv1 wrapper: too short ({} bytes, need ≥{min_len})", bytes.len() )); } - if &bytes[0..4] != WRAPPER_MAGIC { + if &bytes[..magic_len] != WRAPPER_MAGIC { return Err(format!( - "ASK1 wrapper: bad magic {:?}, expected b\"ASK1\"", - &bytes[0..4] + "ASAPv1 wrapper: bad magic {:?}, expected b\"ASAPv1\"", + &bytes[..magic_len] )); } - let version = bytes[4]; + let version = bytes[version_offset]; if version != WRAPPER_VERSION { - return Err(format!("ASK1 wrapper: unsupported version 0x{version:02x}")); + return Err(format!( + "ASAPv1 wrapper: unsupported version 0x{version:02x}" + )); } - let kind_id_len = bytes[5] as usize; - let header_end = 6 + kind_id_len; + let kind_id_len = bytes[kind_id_len_offset] as usize; + let header_end = kind_id_offset + kind_id_len; if bytes.len() < header_end { return Err(format!( - "ASK1 wrapper: kind_id_len={kind_id_len} but only {} bytes available after offset 6", - bytes.len().saturating_sub(6) + "ASAPv1 wrapper: kind_id_len={kind_id_len} but only {} bytes available after offset {kind_id_offset}", + bytes.len().saturating_sub(kind_id_offset) )); } - Ok((&bytes[6..header_end], &bytes[header_end..])) + Ok((&bytes[kind_id_offset..header_end], &bytes[header_end..])) } /// HLL sketch (all variants: Regular, Datafusion, Hip). diff --git a/src/message_pack_format/portable/ddsketch.rs b/src/message_pack_format/portable/ddsketch.rs index 7917ac6..c5cbc02 100644 --- a/src/message_pack_format/portable/ddsketch.rs +++ b/src/message_pack_format/portable/ddsketch.rs @@ -549,14 +549,14 @@ mod tests { /// element count so the bytes stay parity-aligned with the Go /// reference implementation. /// - /// The binary is wrapped in the ASK1 envelope; the payload starts after - /// the header (b"ASK1" + version + kind_id_len + kind_id). + /// The binary is wrapped in the ASAPv1 envelope; the payload starts after + /// the header (b"ASAPv1" + version + kind_id_len + kind_id). #[test] fn test_msgpack_is_three_element_array() { use crate::message_pack_format::magic_ids; let sk = DdSketch::from_raw(0.01, vec![1, 2, 3], -2); let bytes = sk.to_msgpack().unwrap(); - let (kind_id, payload) = magic_ids::decode_wrapper(&bytes).expect("ASK1 header"); + let (kind_id, payload) = magic_ids::decode_wrapper(&bytes).expect("ASAPv1 header"); assert_eq!( kind_id, [magic_ids::DD_SKETCH], diff --git a/src/message_pack_format/portable/set_aggregator.rs b/src/message_pack_format/portable/set_aggregator.rs index 0c49c48..99a8cde 100644 --- a/src/message_pack_format/portable/set_aggregator.rs +++ b/src/message_pack_format/portable/set_aggregator.rs @@ -166,7 +166,7 @@ mod tests { let mut sa = SetAggregator::new(); sa.update("a"); let bytes = sa.to_msgpack().unwrap(); - let (_, payload) = magic_ids::decode_wrapper(&bytes).expect("ASK1 header"); + let (_, payload) = magic_ids::decode_wrapper(&bytes).expect("ASAPv1 header"); let decoded: StringSet = rmp_serde::from_slice(payload).expect("should decode as StringSet { values: ... }"); assert!(decoded.values.contains("a")); diff --git a/src/sketch_framework/hydra.rs b/src/sketch_framework/hydra.rs index c7db0a3..f90ee0a 100644 --- a/src/sketch_framework/hydra.rs +++ b/src/sketch_framework/hydra.rs @@ -160,7 +160,7 @@ impl Hydra { self.query_key(key, &HydraQuery::Cdf(threshold)) } - /// Serializes the Hydra sketch into ASK1-wrapped MessagePack bytes. + /// Serializes the Hydra sketch into ASAPv1-wrapped MessagePack bytes. /// kind_id: `[NATIVE_HYDRA, HASHER_UNKNOWN]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; diff --git a/src/sketch_framework/univmon.rs b/src/sketch_framework/univmon.rs index b08dd76..d52cd5e 100644 --- a/src/sketch_framework/univmon.rs +++ b/src/sketch_framework/univmon.rs @@ -265,7 +265,7 @@ impl UnivMon { &mut self.hh_layers[layer] } - /// Serializes the UnivMon sketch into ASK1-wrapped MessagePack bytes. + /// Serializes the UnivMon sketch into ASAPv1-wrapped MessagePack bytes. /// kind_id: `[NATIVE_UNIVMON, HASHER_UNKNOWN]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; diff --git a/src/sketches/countminsketch.rs b/src/sketches/countminsketch.rs index caa628d..888625d 100644 --- a/src/sketches/countminsketch.rs +++ b/src/sketches/countminsketch.rs @@ -266,7 +266,7 @@ impl CountMin { // Wire layout: [ mode_id: u8 | hasher_id: u8 | ] impl CountMin { - /// Serializes the sketch into ASK1-wrapped MessagePack bytes. + /// Serializes the sketch into ASAPv1-wrapped MessagePack bytes. /// kind_id: `[NATIVE_COUNT_MIN_REGULAR, hasher_id]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; @@ -299,7 +299,7 @@ impl Deserialize<'de>, H: SketchHasher> CountMin CountMin { - /// Serializes the sketch into ASK1-wrapped MessagePack bytes. + /// Serializes the sketch into ASAPv1-wrapped MessagePack bytes. /// kind_id: `[NATIVE_COUNT_MIN_FAST, hasher_id]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; @@ -1027,7 +1027,7 @@ mod tests { let encoded = sketch .serialize_to_bytes() .expect("serialize CountMin FastPath"); - let (kind_id, _) = magic_ids::decode_wrapper(&encoded).expect("ASK1 header"); + let (kind_id, _) = magic_ids::decode_wrapper(&encoded).expect("ASAPv1 header"); assert_eq!( kind_id, [ diff --git a/src/sketches/countsketch.rs b/src/sketches/countsketch.rs index 14161ef..162b262 100644 --- a/src/sketches/countsketch.rs +++ b/src/sketches/countsketch.rs @@ -284,7 +284,7 @@ where S: MatrixStorage + Serialize, C: CountSketchCounter, { - /// Serializes the sketch into ASK1-wrapped MessagePack bytes. + /// Serializes the sketch into ASAPv1-wrapped MessagePack bytes. /// kind_id: `[NATIVE_COUNT_SKETCH_REGULAR, hasher_id]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; @@ -325,7 +325,7 @@ where S: MatrixStorage + Serialize, C: CountSketchCounter, { - /// Serializes the sketch into ASK1-wrapped MessagePack bytes. + /// Serializes the sketch into ASAPv1-wrapped MessagePack bytes. /// kind_id: `[NATIVE_COUNT_SKETCH_FAST, hasher_id]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; @@ -1175,7 +1175,7 @@ mod tests { let encoded = sketch .serialize_to_bytes() .expect("serialize Count FastPath"); - let (kind_id, _) = magic_ids::decode_wrapper(&encoded).expect("ASK1 header"); + let (kind_id, _) = magic_ids::decode_wrapper(&encoded).expect("ASAPv1 header"); assert_eq!( kind_id, [ diff --git a/src/sketches/countsketch_topk.rs b/src/sketches/countsketch_topk.rs index 4798e9e..0928bea 100644 --- a/src/sketches/countsketch_topk.rs +++ b/src/sketches/countsketch_topk.rs @@ -1106,7 +1106,7 @@ impl CountL2HH { compute_median_inline_f64(&mut lst[..]) } - /// Serializes the CountL2HH sketch into ASK1-wrapped MessagePack bytes. + /// Serializes the CountL2HH sketch into ASAPv1-wrapped MessagePack bytes. /// kind_id: `[NATIVE_CMS_HEAP, hasher_id]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; diff --git a/src/sketches/ddsketch.rs b/src/sketches/ddsketch.rs index 4c938e1..347421e 100644 --- a/src/sketches/ddsketch.rs +++ b/src/sketches/ddsketch.rs @@ -165,7 +165,7 @@ impl DDSketch { } } - /// Serializes the sketch to ASK1-wrapped MessagePack bytes. + /// Serializes the sketch to ASAPv1-wrapped MessagePack bytes. /// kind_id: `[NATIVE_DD_SKETCH, HASHER_UNKNOWN]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; diff --git a/src/sketches/hll.rs b/src/sketches/hll.rs index c18b3ce..7c4fad7 100644 --- a/src/sketches/hll.rs +++ b/src/sketches/hll.rs @@ -166,7 +166,7 @@ impl impl HyperLogLogImpl { - /// Serializes the sketch into ASK1-wrapped MessagePack bytes. + /// Serializes the sketch into ASAPv1-wrapped MessagePack bytes. /// kind_id: `[NATIVE_HLL_CLASSIC, hasher_id]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; @@ -203,7 +203,7 @@ impl Deserialize<'de>, H: SketchHasher> impl HyperLogLogImpl { - /// Serializes the sketch into ASK1-wrapped MessagePack bytes. + /// Serializes the sketch into ASAPv1-wrapped MessagePack bytes. /// kind_id: `[NATIVE_HLL_ERTL_MLE, hasher_id]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; @@ -436,7 +436,7 @@ impl HyperLogLogHIPImpl { self.est as usize } - /// Serializes the sketch into ASK1-wrapped MessagePack bytes. + /// Serializes the sketch into ASAPv1-wrapped MessagePack bytes. /// kind_id: `[NATIVE_HLL_HIP, HASHER_DEFAULT_XX]` — HIP always uses DefaultXxHasher. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; @@ -778,8 +778,9 @@ mod tests { .serialize_to_bytes() .unwrap(); - let (classic_kind_id, _) = magic_ids::decode_wrapper(&classic_bytes).expect("ASK1 header"); - let (ertl_kind_id, _) = magic_ids::decode_wrapper(&ertl_bytes).expect("ASK1 header"); + let (classic_kind_id, _) = + magic_ids::decode_wrapper(&classic_bytes).expect("ASAPv1 header"); + let (ertl_kind_id, _) = magic_ids::decode_wrapper(&ertl_bytes).expect("ASAPv1 header"); assert_eq!(classic_kind_id[0], magic_ids::NATIVE_HLL_CLASSIC); assert_eq!(ertl_kind_id[0], magic_ids::NATIVE_HLL_ERTL_MLE); diff --git a/src/sketches/kll.rs b/src/sketches/kll.rs index b7b7203..9f5d6fa 100644 --- a/src/sketches/kll.rs +++ b/src/sketches/kll.rs @@ -662,7 +662,7 @@ impl KLL { // -- Serialization ------------------------------------------------------- - /// Serializes the sketch into ASK1-wrapped MessagePack bytes. + /// Serializes the sketch into ASAPv1-wrapped MessagePack bytes. /// kind_id: `[NATIVE_KLL, HASHER_UNKNOWN]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> where diff --git a/src/sketches/kll_dynamic.rs b/src/sketches/kll_dynamic.rs index 5f381aa..c7a4885 100644 --- a/src/sketches/kll_dynamic.rs +++ b/src/sketches/kll_dynamic.rs @@ -353,7 +353,7 @@ impl KLLDynamic { self.items.len() } - /// Serialize the sketch into ASK1-wrapped MessagePack bytes. + /// Serialize the sketch into ASAPv1-wrapped MessagePack bytes. /// kind_id: `[NATIVE_KLL_DYNAMIC, HASHER_UNKNOWN]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> where diff --git a/src/sketches/kmv.rs b/src/sketches/kmv.rs index 13f5e7a..57f792c 100644 --- a/src/sketches/kmv.rs +++ b/src/sketches/kmv.rs @@ -77,7 +77,7 @@ impl KMV { } } - /// Serializes the sketch into ASK1-wrapped MessagePack bytes. + /// Serializes the sketch into ASAPv1-wrapped MessagePack bytes. /// kind_id: `[NATIVE_KMV, H::hasher_magic_id()]`. pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { use crate::message_pack_format::magic_ids; From a5f7fb2a16a35cb45d3feb630052a36d6c5aec50 Mon Sep 17 00:00:00 2001 From: GordonYuanyc Date: Wed, 8 Jul 2026 15:13:15 -0400 Subject: [PATCH 11/11] feat(msgpack): record hash metadata in wrapper --- docs/msgpack-magic-ids.md | 35 +++- src/message_pack_format/magic_ids.rs | 232 ++++++++++++++++++++++++++- 2 files changed, 254 insertions(+), 13 deletions(-) diff --git a/docs/msgpack-magic-ids.md b/docs/msgpack-magic-ids.md index 2eec6ba..099b5a6 100644 --- a/docs/msgpack-magic-ids.md +++ b/docs/msgpack-magic-ids.md @@ -2,22 +2,45 @@ Every serialised sketch binary produced by this library is wrapped in the **ASAPv1 envelope**, a self-describing header that carries the sketch's -type discriminant without a fixed-size ceiling on the number of types. +type discriminant plus wrapper-level metadata needed by consumers that update +or query sketches outside the original process. ``` -┌───────────────┬────────────┬──────────────────┬───────────────────────┬──────────────────────┐ -│ b"ASAPv1": 6B │ version: u8 │ kind_id_len: u8 │ kind_id: [kind_id_len] │ msgpack payload … │ -└───────────────┴────────────┴──────────────────┴───────────────────────┴──────────────────────┘ +┌───────────────┬────────────┬──────────────────┬───────────────────────┬───────────────────┬────────────────────┬──────────────────────┐ +│ b"ASAPv1": 6B │ version: u8 │ kind_id_len: u8 │ kind_id: [kind_id_len] │ metadata_len: u32 │ metadata: msgpack │ msgpack payload … │ +└───────────────┴────────────┴──────────────────┴───────────────────────┴───────────────────┴────────────────────┴──────────────────────┘ ``` | Field | Value | Notes | |-------|-------|-------| | `b"ASAPv1"` | `0x41 0x53 0x41 0x50 0x76 0x31` | 6-byte ASCII sentinel, not a valid msgpack prefix | -| `version` | `0x01` | Increment only if the envelope layout changes | +| `version` | `0x02` | Increment only if the envelope layout changes | | `kind_id_len` | 1 or 2 | Number of `kind_id` bytes (1 for portable, 2 for native) | | `kind_id` | see tables below | Canonical big-endian, no leading zero bytes | +| `metadata_len` | big-endian `u32` | Number of bytes in the metadata MessagePack block | +| `metadata` | compact msgpack array | Wrapper metadata; currently records the hash profile | | payload | msgpack bytes | Compact (array) or named (map) depending on sketch type | +Current metadata fields, in compact MessagePack array order: + +| Index | Field | Standard hash value | +|-------|-------|---------------------| +| 0 | `metadata_version` | `1` | +| 1 | `hash_spec_present` | `true` for portable IDs and registered `DefaultXxHasher` native IDs | +| 2 | `hash_profile_id` | `projectasap.xxh3.seedlist.v1` | +| 3 | `hash_algorithm` | `xxh3_64_128` | +| 4 | `seed_list` | `[0xcafe3553, 0xade3415118, 0x8cc70208, 0x2f024b2b, 0x451a3df5, 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, 0xcbbb9d5d, 0x629a292a, 0x9159015a, 0x152fecd8, 0x67332667, 0x8eb44a87, 0xdb0c2e0d]` | +| 5 | `canonical_seed_index` | `5` | +| 6 | `matrix_seed_index` | `0` | +| 7 | `hydra_seed_index` | `6` | +| 8 | `univmon_bottom_layer_seed_index` | `19` | +| 9 | `seed_derivation` | `seed_list_index_wrap` | +| 10 | `input_encoding` | `projectasap.input.v1` | + +If a wrapper cannot declare a complete registered hash profile, it still carries +metadata version `1` with `hash_spec_present = false` and empty/default hash +fields. Readers validate known hash metadata before exposing the payload. + **Portable** sketches use a 1-byte `kind_id` (`kind_id_len = 1`). **Native** Rust sketches use a 2-byte `kind_id` (`kind_id_len = 2`): first byte is the type/mode discriminant, second byte is the hasher ID (`HASHER_*`). @@ -98,7 +121,7 @@ bytes are produced by `KLL::serialize_to_bytes` and therefore carry the native `0x8a` prefix. The portable round-trip is: ``` -KllSketch::to_msgpack() → [ ASAPv1 | v=1 | len=1 | 0x06 | msgpack([k, [ASAPv1|v=1|len=2|0x8a|0xff|raw_kll]]) ] +KllSketch::to_msgpack() → [ ASAPv1 | v=1 | len=1 | 0x06 | metadata | msgpack([k, [ASAPv1|v=1|len=2|0x8a|0xff|metadata|raw_kll]]) ] KllSketch::from_msgpack() → decode_wrapper → kind_id=[0x06], payload=msgpack struct → decode struct: k + sketch_bytes → KLL::deserialize_from_bytes(sketch_bytes) diff --git a/src/message_pack_format/magic_ids.rs b/src/message_pack_format/magic_ids.rs index fa78768..65fee54 100644 --- a/src/message_pack_format/magic_ids.rs +++ b/src/message_pack_format/magic_ids.rs @@ -5,7 +5,7 @@ //! reserves space for future metadata without a fixed-size ceiling: //! //! ```text -//! [ b"ASAPv1" | version: u8 | kind_id_len: u8 | kind_id: [u8; kind_id_len] | ] +//! [ b"ASAPv1" | version: u8 | kind_id_len: u8 | kind_id: [u8; kind_id_len] | metadata_len: u32 | metadata: bytes | ] //! ``` //! //! * `b"ASAPv1"` — 6-byte ASCII sentinel; unambiguously not a msgpack value. @@ -14,6 +14,8 @@ //! canonical unsigned big-endian with no leading zero bytes. For current //! sketch types this is 1–2 bytes; future additions can use more without a //! protocol change. +//! * `metadata_len + metadata` — MessagePack metadata describing the hash +//! profile needed for cross-process updates and queries. //! //! **Portable** sketches (cross-language, shared with `sketchlib-go`) use a //! 1-byte `kind_id` drawn from the `0x01–0x09` range. @@ -33,17 +35,119 @@ pub const WRAPPER_MAGIC: &[u8; 6] = b"ASAPv1"; /// Envelope layout version stored immediately after `WRAPPER_MAGIC`. Increment /// if the header structure (field order, field semantics) ever changes. -pub const WRAPPER_VERSION: u8 = 0x01; +pub const WRAPPER_VERSION: u8 = 0x02; + +use serde::{Deserialize, Serialize}; + +pub const HASH_PROFILE_PROJECTASAP_XXH3_V1: &str = "projectasap.xxh3.seedlist.v1"; +pub const HASH_ALGORITHM_XXH3_64_128: &str = "xxh3_64_128"; +pub const HASH_SEED_DERIVATION_INDEX_WRAP: &str = "seed_list_index_wrap"; +pub const HASH_INPUT_ENCODING_PROJECTASAP_V1: &str = "projectasap.input.v1"; +pub const MATRIX_BASE_SEED_INDEX: u32 = 0; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct WrapperMetadata { + pub metadata_version: u8, + pub hash_spec_present: bool, + pub hash_profile_id: String, + pub hash_algorithm: String, + pub seed_list: Vec, + pub canonical_seed_index: u32, + pub matrix_seed_index: u32, + pub hydra_seed_index: u32, + pub univmon_bottom_layer_seed_index: u32, + pub seed_derivation: String, + pub input_encoding: String, +} + +impl WrapperMetadata { + pub fn standard_hash() -> Self { + Self { + metadata_version: 1, + hash_spec_present: true, + hash_profile_id: HASH_PROFILE_PROJECTASAP_XXH3_V1.to_string(), + hash_algorithm: HASH_ALGORITHM_XXH3_64_128.to_string(), + seed_list: crate::SEEDLIST.to_vec(), + canonical_seed_index: crate::CANONICAL_HASH_SEED as u32, + matrix_seed_index: MATRIX_BASE_SEED_INDEX, + hydra_seed_index: crate::HYDRA_SEED as u32, + univmon_bottom_layer_seed_index: crate::BOTTOM_LAYER_FINDER as u32, + seed_derivation: HASH_SEED_DERIVATION_INDEX_WRAP.to_string(), + input_encoding: HASH_INPUT_ENCODING_PROJECTASAP_V1.to_string(), + } + } + + pub fn no_hash_spec() -> Self { + Self { + metadata_version: 1, + hash_spec_present: false, + hash_profile_id: String::new(), + hash_algorithm: String::new(), + seed_list: Vec::new(), + canonical_seed_index: 0, + matrix_seed_index: 0, + hydra_seed_index: 0, + univmon_bottom_layer_seed_index: 0, + seed_derivation: String::new(), + input_encoding: String::new(), + } + } + + fn validate(&self) -> Result<(), String> { + if self.metadata_version != 1 { + return Err(format!( + "ASAPv1 wrapper: unsupported metadata_version {}", + self.metadata_version + )); + } + if !self.hash_spec_present { + let expected = Self::no_hash_spec(); + if self != &expected { + return Err("ASAPv1 wrapper: no-hash metadata mismatch".to_string()); + } + return Ok(()); + } + let expected = Self::standard_hash(); + if self != &expected { + return Err("ASAPv1 wrapper: hash metadata mismatch".to_string()); + } + Ok(()) + } +} + +fn metadata_for_kind_id(kind_id: &[u8]) -> WrapperMetadata { + match kind_id { + // Portable wire formats are cross-language and always use the standard + // ProjectASAP hash profile when they need key hashing. Keep the profile + // present for every portable kind so consumers can validate uniformly. + [0x01..=0x09] => WrapperMetadata::standard_hash(), + // Native Rust sketches with an explicit DefaultXxHasher byte. + [_, HASHER_DEFAULT_XX] => WrapperMetadata::standard_hash(), + // Native framework sketches with no generic hasher but standard hash use. + [NATIVE_HYDRA | NATIVE_UNIVMON, HASHER_UNKNOWN] => WrapperMetadata::standard_hash(), + // DDSketch/KLL and custom/unknown hashers do not have a complete, + // registered hash profile to record. + _ => WrapperMetadata::no_hash_spec(), + } +} /// Prepend the ASAPv1 envelope to `payload` and return the complete binary. /// /// `kind_id` identifies the sketch type (1–2 bytes for all current types). pub fn encode_wrapper(kind_id: &[u8], payload: &[u8]) -> Vec { - let mut out = Vec::with_capacity(WRAPPER_MAGIC.len() + 1 + 1 + kind_id.len() + payload.len()); + let metadata = metadata_for_kind_id(kind_id); + let metadata_bytes = + rmp_serde::to_vec(&metadata).expect("WrapperMetadata serialization cannot fail"); + let metadata_len = u32::try_from(metadata_bytes.len()).expect("wrapper metadata too large"); + let mut out = Vec::with_capacity( + WRAPPER_MAGIC.len() + 1 + 1 + kind_id.len() + 4 + metadata_bytes.len() + payload.len(), + ); out.extend_from_slice(WRAPPER_MAGIC); out.push(WRAPPER_VERSION); out.push(kind_id.len() as u8); out.extend_from_slice(kind_id); + out.extend_from_slice(&metadata_len.to_be_bytes()); + out.extend_from_slice(&metadata_bytes); out.extend_from_slice(payload); out } @@ -53,11 +157,21 @@ pub fn encode_wrapper(kind_id: &[u8], payload: &[u8]) -> Vec { /// Returns `Err(String)` on any structural mismatch so callers can convert to /// their own error type. pub fn decode_wrapper(bytes: &[u8]) -> Result<(&[u8], &[u8]), String> { + let (kind_id, _metadata, payload) = decode_wrapper_with_metadata(bytes)?; + Ok((kind_id, payload)) +} + +/// Strip the ASAPv1 envelope from `bytes` and return `(kind_id, metadata, payload)`. +/// +/// Returns `Err(String)` on any structural mismatch or unsupported metadata. +pub fn decode_wrapper_with_metadata( + bytes: &[u8], +) -> Result<(&[u8], WrapperMetadata, &[u8]), String> { let magic_len = WRAPPER_MAGIC.len(); let version_offset = magic_len; let kind_id_len_offset = magic_len + 1; let kind_id_offset = magic_len + 2; - let min_len = kind_id_offset + 1; + let min_len = kind_id_offset + 1 + 4; if bytes.len() < min_len { return Err(format!( @@ -78,14 +192,118 @@ pub fn decode_wrapper(bytes: &[u8]) -> Result<(&[u8], &[u8]), String> { )); } let kind_id_len = bytes[kind_id_len_offset] as usize; - let header_end = kind_id_offset + kind_id_len; - if bytes.len() < header_end { + let metadata_len_offset = kind_id_offset + kind_id_len; + if bytes.len() < metadata_len_offset + 4 { return Err(format!( "ASAPv1 wrapper: kind_id_len={kind_id_len} but only {} bytes available after offset {kind_id_offset}", bytes.len().saturating_sub(kind_id_offset) )); } - Ok((&bytes[kind_id_offset..header_end], &bytes[header_end..])) + let metadata_len = u32::from_be_bytes( + bytes[metadata_len_offset..metadata_len_offset + 4] + .try_into() + .expect("metadata length slice has four bytes"), + ) as usize; + let metadata_start = metadata_len_offset + 4; + let metadata_end = metadata_start + metadata_len; + if bytes.len() < metadata_end { + return Err(format!( + "ASAPv1 wrapper: metadata_len={metadata_len} but only {} bytes available after offset {metadata_start}", + bytes.len().saturating_sub(metadata_start) + )); + } + let metadata: WrapperMetadata = rmp_serde::from_slice(&bytes[metadata_start..metadata_end]) + .map_err(|err| format!("ASAPv1 wrapper: invalid metadata: {err}"))?; + metadata.validate()?; + let kind_id = &bytes[kind_id_offset..metadata_len_offset]; + let expected_metadata = metadata_for_kind_id(kind_id); + if metadata != expected_metadata { + return Err("ASAPv1 wrapper: metadata does not match kind_id".to_string()); + } + Ok((kind_id, metadata, &bytes[metadata_end..])) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn encode_wrapper_with_metadata( + kind_id: &[u8], + metadata: &WrapperMetadata, + payload: &[u8], + ) -> Vec { + let metadata_bytes = rmp_serde::to_vec(metadata).expect("metadata serialization"); + let metadata_len = u32::try_from(metadata_bytes.len()).expect("metadata too large"); + let mut out = Vec::new(); + out.extend_from_slice(WRAPPER_MAGIC); + out.push(WRAPPER_VERSION); + out.push(kind_id.len() as u8); + out.extend_from_slice(kind_id); + out.extend_from_slice(&metadata_len.to_be_bytes()); + out.extend_from_slice(&metadata_bytes); + out.extend_from_slice(payload); + out + } + + #[test] + fn portable_wrapper_records_standard_hash_metadata() { + let payload = [0x91, 0x2a]; + let encoded = encode_wrapper(&[COUNT_MIN_SKETCH], &payload); + + let (kind_id, metadata, decoded_payload) = + decode_wrapper_with_metadata(&encoded).expect("decode wrapper"); + + assert_eq!(kind_id, &[COUNT_MIN_SKETCH]); + assert_eq!(decoded_payload, payload); + assert_eq!(metadata, WrapperMetadata::standard_hash()); + assert_eq!(metadata.seed_list, crate::SEEDLIST); + assert_eq!( + metadata.canonical_seed_index, + crate::CANONICAL_HASH_SEED as u32 + ); + assert_eq!(metadata.hydra_seed_index, crate::HYDRA_SEED as u32); + assert_eq!( + metadata.univmon_bottom_layer_seed_index, + crate::BOTTOM_LAYER_FINDER as u32 + ); + } + + #[test] + fn unknown_native_hasher_does_not_claim_hash_metadata() { + let payload = [0x90]; + let kind_id = [NATIVE_COUNT_MIN_REGULAR, HASHER_UNKNOWN]; + let encoded = encode_wrapper(&kind_id, &payload); + + let (decoded_kind_id, metadata, decoded_payload) = + decode_wrapper_with_metadata(&encoded).expect("decode wrapper"); + + assert_eq!(decoded_kind_id, kind_id); + assert_eq!(decoded_payload, payload); + assert_eq!(metadata, WrapperMetadata::no_hash_spec()); + } + + #[test] + fn wrapper_rejects_hash_metadata_mismatch() { + let payload = [0x90]; + let mut metadata = WrapperMetadata::standard_hash(); + metadata.canonical_seed_index += 1; + let encoded = encode_wrapper_with_metadata(&[COUNT_SKETCH], &metadata, &payload); + + let err = decode_wrapper(&encoded).expect_err("metadata mismatch should fail"); + + assert!(err.contains("hash metadata mismatch"), "{err}"); + } + + #[test] + fn wrapper_rejects_metadata_kind_id_mismatch() { + let payload = [0x90]; + let metadata = WrapperMetadata::no_hash_spec(); + let encoded = encode_wrapper_with_metadata(&[COUNT_SKETCH], &metadata, &payload); + + let err = decode_wrapper(&encoded).expect_err("metadata kind mismatch should fail"); + + assert!(err.contains("metadata does not match kind_id"), "{err}"); + } } /// HLL sketch (all variants: Regular, Datafusion, Hip).