diff --git a/Cargo.lock b/Cargo.lock index 9b91ce1..73597b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,6 +20,7 @@ dependencies = [ "rmp-serde", "serde", "serde-big-array", + "serde_bytes", "smallvec", "twox-hash", "xxhash-rust", @@ -265,6 +266,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.228" diff --git a/Cargo.toml b/Cargo.toml index 150820c..b52a296 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ all-features = true rand = "0.9.3" serde = { version = "1.0", features = ["derive"] } serde-big-array = "0.5" +serde_bytes = "0.11" rmp-serde = "1.3.0" twox-hash = "2.1.2" smallvec = "1.13.2" diff --git a/asapv1_golden/README.md b/asapv1_golden/README.md new file mode 100644 index 0000000..cc39625 --- /dev/null +++ b/asapv1_golden/README.md @@ -0,0 +1,49 @@ +# ASAPv1 cross-language golden byte-vectors + +These `.hex` files pin the exact bytes the ASAPv1 wire format (see +`docs/asapv1_wire_format.md`) emits for a set of fixed, known sketch states. They +are the machine-checked proof that the Rust (`asap_sketchlib`) and Go +(`sketchlib-go`) implementations serialize **byte-identically**. + +**The copy here and the copy in `sketchlib-go/asapv1_golden/` MUST stay +byte-identical.** They are the same fixtures, checked into both repos so each +side's test suite is self-contained. The bytes are authored by the Rust side +(rmp_serde is the reference encoder); Go conforms to them, never the reverse. + +Each file is one line of lowercase hex (no `0x`, no whitespace) = the complete +ASAPv1 envelope `[ magic | version | kind_id | metadata_len | payload_len | +metadata | payload ]`. + +## Design principle: state is fixed, not hashed + +Every fixture is built from a **known raw sketch state** (specific register +bytes / matrix values set directly), never by hashing input values. So the +golden tests the **wire encoding**, isolated from the hash functions. + +## Fixtures + +| File | Sketch | kind_id | State | +| ---- | ------ | ------- | ----- | +| `hll_classic_p12` | HLL Classic, P12 | `01 01` | 4096 registers, set: `[0]=1, [1]=7, [100]=42, [4095]=3` | +| `hll_ertl_mle_p12` | HLL Ertl-MLE, P12 | `01 02` | same register pattern | +| `hll_hip_p12` | HLL HIP, P12 | `01 03` | same registers + `hip_kxq0=1.5, hip_kxq1=2.5, hip_est=3.0` | +| `cms_i64_regular_2x3` | Count-Min i64, RegularPath | `02 00` | 2×3 row-major `[[0,1,127],[128,300,65536]]` | +| `cms_f64_fast_2x3` | Count-Min f64, FastPath | `02 00` | 2×3 row-major `[[0.0,1.5,2.25],[3.75,4.125,5.0625]]` | + +The i64 fixture deliberately spans the msgpack integer width boundaries +(positive fixint / uint8 / uint16 / uint32) to lock the "non-negative integer → +uint family, minimal width" rule (`docs/asapv1_wire_format.md` §5). + +## Tests that consume these + +- Rust: `tests/asapv1_golden.rs` — builds each fixture from known state, + serializes, asserts `== golden`; and asserts `deserialize(golden)` round-trips. +- Go: `wire/asapmsgpack/golden_test.go` — `Unmarshal(golden)`→re-`Marshal` == + golden, **and** `Marshal(equivalent known state) == golden` (the cross-language + parity proof). + +## Regenerating + +If the wire format intentionally changes, regenerate from the Rust side (the +reference encoder) and copy the files into both repos. Both test suites must then +pass unchanged. diff --git a/asapv1_golden/cms_f64_fast_2x3.hex b/asapv1_golden/cms_f64_fast_2x3.hex new file mode 100644 index 0000000..047c47c --- /dev/null +++ b/asapv1_golden/cms_f64_fast_2x3.hex @@ -0,0 +1 @@ +4153415076310102020000000153000000388bb06d657461646174615f76657273696f6e01af686173685f70726f66696c655f6964bc70726f6a656374617361702e787868332e736565646c6973742e7631ae686173685f616c676f726974686dab787868335f36345f313238af736565645f64657269766174696f6eb4736565645f6c6973745f696e6465785f77726170ae696e7075745f656e636f64696e67b470726f6a656374617361702e696e7075742e7631a9736565645f6c697374dc0014cecafe3553cf000000ade3415118ce8cc70208ce2f024b2bce451a3df5ce6a09e667cebb67ae85ce3c6ef372cea54ff53ace510e527fce9b05688cce1f83d9abce5be0cd19cecbbb9d5dce629a292ace9159015ace152fecd8ce67332667ce8eb44a87cedb0c2e0db16d61747269785f736565645f696e64657800a4726f777302a4636f6c7303ac636f756e7465725f74797065a3663634a46d6f6465a4666173749196cb0000000000000000cb3ff8000000000000cb4002000000000000cb400e000000000000cb4010800000000000cb4014400000000000 diff --git a/asapv1_golden/cms_i64_regular_2x3.hex b/asapv1_golden/cms_i64_regular_2x3.hex new file mode 100644 index 0000000..f50baf2 --- /dev/null +++ b/asapv1_golden/cms_i64_regular_2x3.hex @@ -0,0 +1 @@ +41534150763101020200000001560000000f8bb06d657461646174615f76657273696f6e01af686173685f70726f66696c655f6964bc70726f6a656374617361702e787868332e736565646c6973742e7631ae686173685f616c676f726974686dab787868335f36345f313238af736565645f64657269766174696f6eb4736565645f6c6973745f696e6465785f77726170ae696e7075745f656e636f64696e67b470726f6a656374617361702e696e7075742e7631a9736565645f6c697374dc0014cecafe3553cf000000ade3415118ce8cc70208ce2f024b2bce451a3df5ce6a09e667cebb67ae85ce3c6ef372cea54ff53ace510e527fce9b05688cce1f83d9abce5be0cd19cecbbb9d5dce629a292ace9159015ace152fecd8ce67332667ce8eb44a87cedb0c2e0db16d61747269785f736565645f696e64657800a4726f777302a4636f6c7303ac636f756e7465725f74797065a3693634a46d6f6465a7726567756c6172919600017fcc80cd012cce00010000 diff --git a/asapv1_golden/hll_classic_p12.hex b/asapv1_golden/hll_classic_p12.hex new file mode 100644 index 0000000..4e63148 --- /dev/null +++ b/asapv1_golden/hll_classic_p12.hex @@ -0,0 +1 @@ +415341507631010201010000013a0000100488b06d657461646174615f76657273696f6e01af686173685f70726f66696c655f6964bc70726f6a656374617361702e787868332e736565646c6973742e7631ae686173685f616c676f726974686dab787868335f36345f313238af736565645f64657269766174696f6eb4736565645f6c6973745f696e6465785f77726170ae696e7075745f656e636f64696e67b470726f6a656374617361702e696e7075742e7631a9736565645f6c697374dc0014cecafe3553cf000000ade3415118ce8cc70208ce2f024b2bce451a3df5ce6a09e667cebb67ae85ce3c6ef372cea54ff53ace510e527fce9b05688cce1f83d9abce5be0cd19cecbbb9d5dce629a292ace9159015ace152fecd8ce67332667ce8eb44a87cedb0c2e0db463616e6f6e6963616c5f736565645f696e64657805a9707265636973696f6e0c91c51000010700000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003 diff --git a/asapv1_golden/hll_ertl_mle_p12.hex b/asapv1_golden/hll_ertl_mle_p12.hex new file mode 100644 index 0000000..f193b87 --- /dev/null +++ b/asapv1_golden/hll_ertl_mle_p12.hex @@ -0,0 +1 @@ +415341507631010201020000013a0000100488b06d657461646174615f76657273696f6e01af686173685f70726f66696c655f6964bc70726f6a656374617361702e787868332e736565646c6973742e7631ae686173685f616c676f726974686dab787868335f36345f313238af736565645f64657269766174696f6eb4736565645f6c6973745f696e6465785f77726170ae696e7075745f656e636f64696e67b470726f6a656374617361702e696e7075742e7631a9736565645f6c697374dc0014cecafe3553cf000000ade3415118ce8cc70208ce2f024b2bce451a3df5ce6a09e667cebb67ae85ce3c6ef372cea54ff53ace510e527fce9b05688cce1f83d9abce5be0cd19cecbbb9d5dce629a292ace9159015ace152fecd8ce67332667ce8eb44a87cedb0c2e0db463616e6f6e6963616c5f736565645f696e64657805a9707265636973696f6e0c91c51000010700000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003 diff --git a/asapv1_golden/hll_hip_p12.hex b/asapv1_golden/hll_hip_p12.hex new file mode 100644 index 0000000..2b4e02d --- /dev/null +++ b/asapv1_golden/hll_hip_p12.hex @@ -0,0 +1 @@ +415341507631010201030000013a0000101f88b06d657461646174615f76657273696f6e01af686173685f70726f66696c655f6964bc70726f6a656374617361702e787868332e736565646c6973742e7631ae686173685f616c676f726974686dab787868335f36345f313238af736565645f64657269766174696f6eb4736565645f6c6973745f696e6465785f77726170ae696e7075745f656e636f64696e67b470726f6a656374617361702e696e7075742e7631a9736565645f6c697374dc0014cecafe3553cf000000ade3415118ce8cc70208ce2f024b2bce451a3df5ce6a09e667cebb67ae85ce3c6ef372cea54ff53ace510e527fce9b05688cce1f83d9abce5be0cd19cecbbb9d5dce629a292ace9159015ace152fecd8ce67332667ce8eb44a87cedb0c2e0db463616e6f6e6963616c5f736565645f696e64657805a9707265636973696f6e0c94c51000010700000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002a0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003cb3ff8000000000000cb4004000000000000cb4008000000000000 diff --git a/docs/asapv1_wire_format.md b/docs/asapv1_wire_format.md new file mode 100644 index 0000000..880a3bc --- /dev/null +++ b/docs/asapv1_wire_format.md @@ -0,0 +1,488 @@ +# ASAPv1 Wire Format: Design Doc + +## What this is + +ASAPv1 is ProjectASAP's self-describing binary format to serialize sketches in `asap_sketchlib`. +It fixes the exact bytes written when a sketch (HyperLogLog, Count-Min, and so on) is saved or shipped, so any process or language (Rust today, Go next) can decode it, confirm it was hashed compatibly, and merge or query it. +This doc describes that byte layout at a high level; the byte-exact encoding rules live in Section 4, and the implementation notes (decoder validation, converting an in-memory sketch) live in Section 5. + +## Which parts to read + +If the doc feels long, these sections carry the key points: + +- [**Section 1, Envelope**](#section-1-envelope): the Layout table. +- [**Section 2, Metadata**](#section-2-metadata): the fields table, the hash-spec table, the structural-params table. +- [**Section 3, Payload**](#section-3-payload): the HLL payload and the Count-Min payload. + +## Terms + +- **Envelope**: the header that wraps the metadata and payload (`magic | version | kind_id | length-prefixes`). Sketch-agnostic framing; it is not the whole binary. +- **Metadata**: the descriptor (msgpack map) holding the hash spec (how the sketch was hashed) plus the structural params needed to read the payload. +- **Payload**: the sketch's raw state (registers, counters, and so on), a positional msgpack array. The per-sketch-authored part. +- **kind / kind_id**: an id naming the algorithm (e.g. `0x02 0x00` = Count-Min); 2 bytes today, extendable to more since `kind_id_len` is a byte. The registry maps id to name to payload shape. +- **Hash profile**: the identified set of hash constants (algorithm, seeds, seed indices) a sketch was built with, carried in the metadata so the bytes are truthful (`HashProfile` trait in Rust). +- **seed_list**: the array of hash seeds, carried inline in every sketch's metadata so the bytes self-describe the hash with no registry lookup. +- **wire-eligible**: a sketch config the format can serialize. The wire covers a fixed subset of the freer in-memory types; a config outside that subset is converted to a covered one first (Section 3.2, Section 5). + +## Status + +- **Implementing (Rust).** HLL and Count-Min serialize through the shared `message_pack_format::envelope` module per this spec. +- **Self-describing.** The hash-spec metadata is derived from the hasher's `HashProfile` (read live, never hardcoded), so the bytes truthfully describe how a sketch was hashed; custom hash profiles are supported (Section 2). +- **Byte-level encoding** is pinned in Section 4; the resolved decisions are summarized at the end. +- **`sketchlib-go`** is aligned separately (see Cross-language contract). + +## Layering + +| Layer | Scope | Self-describing? | Owner | Changes when | +| ------- | ------- | ------------------ | ------- | -------------- | +| **Envelope** | frame | yes | one shared module | the framing changes (rare) | +| **Metadata** | descriptor (hash spec + structural params) | yes | one shared module | the hash profile or a sketch's params change | +| **Payload** | one per sketch | **no** | each sketch | that sketch's raw encoding changes | + +Every serialized sketch carries its own envelope, metadata, and payload. +What differs is **who authors each part**: the envelope framing and the metadata schema are shared and sketch-agnostic (one module, identical across all sketches), while the payload layout is defined per sketch. +Values still vary per sketch: each blob carries its own precision, rows/cols, register bytes, and so on. + +```text ++-------------------------------+ +| Envelope | Metadata | Payload | ++-------------------------------+ +``` + +### Guiding principle + +In the byte stream, `kind_id` and `metadata` come **before** the `payload`. +By the time the decoder reaches the payload it already knows both, and together they fix the payload's structure completely. +So the payload carries **raw state only**: no field names, no tag that `kind_id` or the metadata already carries, no derived quantities. +If a payload looks complicated, either the sketch genuinely has that much state, or something derivable/redundant leaked in and should be removed. + +### Structure + +A serialized sketch is one envelope, then one metadata, then one payload. + +```mermaid +erDiagram + SerializedSketch { + bytes envelope "1st, framing" + msgpack_map metadata "2nd, descriptor" + msgpack_array payload "3rd, raw state" + } + SerializedSketch ||--|| ENVELOPE : carries-one + SerializedSketch ||--|| METADATA : carries-one + SerializedSketch ||--|| PAYLOAD : carries-one + PAYLOAD ||--o| HLL_PAYLOAD : kind-0x0101-0x0102 + PAYLOAD ||--o| HLL_HIP_PAYLOAD : kind-0x0103 + PAYLOAD ||--o| COUNTMIN_PAYLOAD : kind-0x0200 + ENVELOPE { + bytes magic + u8 version + u8 kind_id_len + bytes kind_id + u32be metadata_len + u32be payload_len + } + METADATA { + u8 metadata_version + string hash_profile_id + string hash_algorithm + string seed_derivation + string input_encoding + array seed_list + u32 seed_index "hash-spec group, per-sketch" + mixed structural_params "structural group, per-sketch: precision(HLL) or rows+cols+counter_type+mode(CMS)" + } + PAYLOAD { + msgpack_array raw_state + } + HLL_PAYLOAD { + bin registers + } + HLL_HIP_PAYLOAD { + bin registers + f64 hip_kxq0 + f64 hip_kxq1 + f64 hip_est + } + COUNTMIN_PAYLOAD { + array counts + } +``` + +--- + +## Section 1: Envelope + +A flat, sketch-agnostic frame. +It answers, with zero knowledge of the sketch: *is this ours?* (magic), *how do I parse the frame?* (version), *what algorithm?* (kind_id). +The envelope is essentially **constant** across sketches; only `kind_id` and the two length fields differ. + +### Layout + +```md +[ magic:6 | version:u8 | kind_id_len:u8 | kind_id:bytes + | metadata_len:u32_be | payload_len:u32_be + | metadata:msgpack | payload:msgpack ] +``` + +| Field | Type | Value / range | Notes | +| ------- | ------ | --------------- | ------- | +| `magic` | 6 bytes | `41 53 41 50 76 31` = `b"ASAPv1"` | fixed sentinel | +| `version` | u8 | `0x01` | envelope layout version; this doc = `0x01` | +| `kind_id_len` | u8 | `2` today (<=255) | length of `kind_id` | +| `kind_id` | bytes | see registry | which algorithm | +| `metadata_len` | u32 be | varies | byte length of the metadata block | +| `payload_len` | u32 be | varies | byte length of the payload | +| `metadata` | msgpack map | varies | Section 2 | +| `payload` | msgpack array | varies | Section 3 | + +**`payload_len`** makes the envelope a self-delimiting record (needed to ever place a sketch inside a larger container). +`metadata_len` is variable only because the metadata is a variable-length msgpack map (Section 2); the length fields are pure framing and do not depend on the sketch. + +### The `kind_id` scheme + +#### Design choice: `kind_id` refers to **algorithm level** + +Count-Min is **one** kind_id: its counter type (i64/f64) and mode (fast/regular) live in the metadata, so the id stays the same across them. +Classic and Ertl-MLE have byte-identical payloads but are separate ids because `kind_id` also selects the *estimator* to apply. + +#### Design choice: a numeric `kind_id` (string form left open) + +The wire carries the compact `kind_id` and resolves the algorithm name through the registry below; it does not encode a raw string like `"HyperLogLog-Classic"`. +The registry is the single place that maps id to name; the door is left open to switch to a string scheme later if it ever earns its keep. + +#### `kind_id` structure + +Today `kind_id` is `[family, variant]` and names the sketch's **algorithm** (its parameters live in the metadata): + +- **family** (byte 1) picks the sketch type: `0x01` = HLL, `0x02` = Count-Min, and so on. +- **variant** (byte 2) picks the algorithm within that family; for HLL, Classic vs Ertl-MLE vs HIP. + +**Allocation rules:** + +- `kind_id` is **variable-length** (`kind_id_len` is a u8), so the id space is effectively unbounded; it can keep growing forever, and we will never run out. +- A `kind_id` is **allocated once and never recycled.** When an algorithm is retired, its id stays reserved permanently; reusing a retired number would make a new decoder silently misread old bytes. +- A **new incompatible payload encoding gets a new `kind_id`** (Q-VER: versioning lives in the id, which keeps payloads minimal; the payload has no version field of its own). + +### kind_id registry (single source of truth, mirrored verbatim in `sketchlib-go`) + +The **family** bytes match `sketchlib-go`'s `wire/asapmsgpack/magic_ids.go` verbatim; `0x0a`+ are new allocations for sketches in [`apis.md`](./apis.md) that Go has not assigned yet. +Only the HLL variants and Count-Min have designed payloads today; every other row reserves its family byte with variant `0x00` and payload **TBD**. +Variant sub-ids are not invented ahead of a payload design; a family that later needs several algorithms allocates its variants when it is designed (as HLL did). +This registry is the master list of algorithms still to design payloads for. + +| kind_id | Sketch | Algorithm / variant | Payload | Status | +| --------- | -------- | --------- | --------- | -------- | +| `0x01 0x00` | HLL | Unspecified | - | reserved | +| `0x01 0x01` | HLL | Classic ("Regular") | Section 3.1 | implemented | +| `0x01 0x02` | HLL | Ertl-MLE ("Datafusion") | Section 3.1 | implemented | +| `0x01 0x03` | HLL | HIP | Section 3.1 | implemented | +| `0x02 0x00` | Count-Min | Count-Min | Section 3.2 | implemented | +| `0x03 0x00` | Count-Min-with-heap (CMSHeap) | - | TBD | assigned in Go / payload not designed | +| `0x04 0x00` | Count Sketch | - | TBD | assigned in Go / payload not designed | +| `0x05 0x00` | DDSketch | - | TBD | assigned in Go / payload not designed | +| `0x06 0x00` | KLL | - | TBD | assigned in Go / payload not designed | +| `0x06 0x01` | KLL dynamic | - | TBD | assigned in Go / payload not designed | +| `0x07 0x00` | Hydra-KLL | - | TBD | assigned in Go / payload not designed | +| `0x08 0x00` | SetAggregator | - | TBD | assigned in Go / payload not designed | +| `0x09 0x00` | DeltaResult | - | TBD | assigned in Go / payload not designed | +| `0x0a 0x00` | Count-Sketch-with-heap (CSHeap) | - | TBD | reserved / not designed | +| `0x0b 0x00` | Elastic (`Unstable`) | - | TBD | reserved / not designed | +| `0x0c 0x00` | Coco (`Unstable`) | - | TBD | reserved / not designed | +| `0x0d 0x00` | UniformSampling (`Unstable`) | - | TBD | reserved / not designed | +| `0x0e 0x00` | KMV (`Unstable`) | - | TBD | reserved / not designed | +| `0x0f 0x00` | HashSketchEnsemble | - | TBD | reserved / not designed | +| `0x10 0x00` | UnivMon | - | TBD | reserved / not designed | +| `0x11 0x00` | UnivMon Optimized | - | TBD | reserved / not designed | +| `0x12 0x00` | NitroBatch | - | TBD | reserved / not designed | +| `0x13 0x00` | ExponentialHistogram | - | TBD | reserved / not designed | +| `0x14 0x00` | EHSketchList | - | TBD | reserved / not designed | +| `0x15 0x00` | EHUnivOptimized (`Unstable`) | - | TBD | reserved / not designed | +| `0x16 0x00` | OctoSketch | - | TBD | reserved / not designed | + +**Mapping notes** (mismatches between `apis.md` and Go's `magic_ids.go`): + +- **CMSHeap vs CSHeap.** Go's `MagicCountMinSketchWithHeap` (`0x03`) is the Count-*Min*-with-heap sketch (`apis.md` to CMSHeap). The Count-*Sketch*-with-heap sketch (`apis.md` to CSHeap) is a distinct family and gets a fresh byte (`0x0a`), separate from `0x03`. +- **Hydra.** `apis.md` lists the "Hydra" framework; Go's only Hydra id is `MagicHydraKLLSketch` (`0x07`), so Hydra maps here to the Hydra-KLL id. If Hydra is later wrapped around a non-KLL base sketch, that combination gets its own id. +- **SetAggregator / DeltaResult** (`0x08` / `0x09`) come from Go's `magic_ids.go` and do not appear as sketches in `apis.md` (they are aggregation and delta-result envelopes, distinct from stand-alone sketches). They are kept here so the family space stays mirrored verbatim with Go. +- **`Unstable`** rows mirror the `Unstable` status those sketches carry in `apis.md`; their kind_id is reserved but the payload (and the sketch API) may still change. + +### Decoder rules + +1. `len >= 6+1+1+0+4+4` before reading anything. +2. `magic` matches, else reject. +3. `version` is known, else reject (no best-effort parse). +4. Read `kind_id`; the per-sketch decoder rejects any `kind_id` it does not own. +5. Read `metadata`; validate it against the target type's profile (Section 5, Validation). +6. Cross-check metadata against `kind_id` and the payload, so structural params stay consistent (Section 5, Validation). +7. Read exactly `payload_len` bytes; hand to the per-sketch payload decoder. +8. Fail **closed** on any inconsistency; never merge or query a sketch whose hash spec did not validate. + +> Implementation note: the shared envelope module (`src/message_pack_format/envelope.rs`) owns rules 1-3 and the byte framing (`encode` / `split`); it is sketch-agnostic and does not know the registry. +> Rule 4 (and metadata/kind_id validation) happens in each sketch's decoder, which checks the `kind_id` against the ones it owns. + +--- + +## Section 2: Metadata + +The **descriptor**: the configuration of a sketch algorithm. +Two groups of fields: + +- **Hash spec**: how keys were hashed (so two sketches can be checked mergeable and a query key hashed the same way). Profile-derived. +- **Structural params**: parameters that shape the payload (HLL precision, CMS counter type, CMS mode). Per-sketch, per-algorithm. + +**Simple rule:** anything you configure when creating a sketch goes in the metadata. + +### Encoding: msgpack **map** keyed by field name + +- Metadata is a **msgpack map**, so a consumer reads fields by name (`"hash_profile_id"`) with no positional guesswork. +- The schema is fixed and closed: **each sketch has its own fixed metadata schema** (in Rust, one struct per sketch with `#[serde(deny_unknown_fields)]`). +- The field *set* differs per sketch: HLL carries `precision` + `canonical_seed_index`; Count-Min carries `rows`, `cols`, `counter_type`, `mode` + `matrix_seed_index`. +- Within a given sketch **every field is required**: a missing key, or an unexpected extra key, is **rejected (fail closed)**, never silently defaulted or skipped. +- **`seed_list` is inlined**, so the bytes self-describe the hash; a consumer reads the exact seeds straight from the binary, with no registry. (It costs ~130 bytes; resolving seeds from `hash_profile_id` via a registry is a v2 space optimization.) + +### Fields + +The metadata map is **two groups** of fields, written on the wire in this order: Hash spec first, then Structural params. + +| Group | Role | Fields | +| ----- | ---- | ------ | +| **Hash spec** | how keys were hashed (check mergeability + re-hash a query key) | `metadata_version`, `hash_profile_id`, `hash_algorithm`, `seed_derivation`, `input_encoding`, `seed_list`, + the seed index(es) it uses | +| **Structural params** | parameters that shape the payload | `precision` (HLL); `rows`, `cols`, `counter_type`, `mode` (Count-Min) | + +The two tables below are the field-by-field detail of each group. + +**Hash spec** + +| Key | Type | Required | Meaning | +| ------- | ------ | -------- | --------- | +| `metadata_version` | u8 | yes | schema version of *this block* (`1`). Independent of envelope `version`. | +| `hash_profile_id` | string | yes | stable global id `"projectasap.xxh3.seedlist.v1"`; authoritative | +| `hash_algorithm` | string | yes | `"xxh3_64_128"` | +| `seed_derivation` | string | yes | `"seed_list_index_wrap"` | +| `input_encoding` | string | yes | `"projectasap.input.v1"` | +| `seed_list` | `array` | **yes (inlined)** | the 20 seeds, carried inline so the bytes self-describe the hash | +| `canonical_seed_index` | u32 | **per-sketch** | index into `seed_list` (`5`); HLL uses it | +| `matrix_seed_index` | u32 | **per-sketch** | `0`; Count-Min uses it | +| `hydra_seed_index` | u32 | **per-sketch** | `6`; include only if used | +| `univmon_bottom_layer_seed_index` | u32 | **per-sketch** | `19`; include only if used | + +**Structural params** + +| Key | Type | Applies to | Meaning | +| ------- | ------ | -------- | --------- | +| `precision` | u8 | HLL | `12` / `14` / `16`; register count = `2^precision` | +| `rows` | u32 | Count-Min | matrix depth (number of hash rows) | +| `cols` | u32 | Count-Min | matrix width (number of columns) | +| `counter_type` | string | Count-Min | `"i64"` or `"f64"`; element type of `counts` | +| `mode` | string | Count-Min | `"fast"` or `"regular"`; key-to-column derivation | + +Count-Min's matrix dimensions are **configuration** (they shape the payload, like HLL's `precision`), so per the config-to-metadata rule they live here. +Count-Min's canonical structural-param order is `... matrix_seed_index, rows, cols, counter_type, mode`; this is the wire contract and Go must mirror it verbatim. + +### Standard ProjectASAP profile (reference values) + +The hash-spec field *values* are read live from the hasher's `HashProfile`: `hll_metadata::` / `cms_metadata::` read `PROFILE_ID`, `ALGORITHM`, `SEED_DERIVATION`, `INPUT_ENCODING`, `seed_list()`, and the seed index straight off `H`. +The block below is the **standard profile**, the one `DefaultXxHasher` declares (the single source of truth for these values); it is also what the registry resolves `hash_profile_id` to. +A single sketch's metadata carries `hash_profile_id` plus only the subset of indices/params it uses. + +```md +metadata_version = 1 +hash_profile_id = "projectasap.xxh3.seedlist.v1" +hash_algorithm = "xxh3_64_128" +seed_list = [0xcafe3553, 0xade3415118, 0x8cc70208, 0x2f024b2b, 0x451a3df5, + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, + 0x9b05688c, 0x1f83d9ab, 0x5be0cd19, 0xcbbb9d5d, 0x629a292a, + 0x9159015a, 0x152fecd8, 0x67332667, 0x8eb44a87, 0xdb0c2e0d] +canonical_seed_index = 5 +matrix_seed_index = 0 +hydra_seed_index = 6 +univmon_bottom_layer_seed_index = 19 +seed_derivation = "seed_list_index_wrap" +input_encoding = "projectasap.input.v1" +``` + +### Custom hash profiles + +Because the metadata is `HashProfile`-derived, a hasher that declares its own profile (a different `PROFILE_ID` / `seed_list()` / seed index) serializes **truthfully**; its own values land in the metadata. +Since `seed_list` is inlined, those bytes are **fully self-describing**: a consumer reads the exact seeds and algorithm straight from the binary, with no registry, even for a hash it has never seen. +This is safe on both ends because serialization **fails closed**: + +- **Encode side (compile-time).** `serialize_to_bytes` is bounded on `H: HashProfile`, so a hasher that does *not* declare a profile simply cannot serialize; mislabeled bytes are impossible by construction. +- **Decode side (runtime).** Decode validates the metadata against the *target* type's `HashProfile`, so bytes hashed under profile A cannot be decoded into a profile-B-typed sketch; they are rejected (see Section 5, Validation). +- **Merge.** Merge compatibility is hash-spec equality (same `hash_profile_id` + seeds). A custom-profile sketch is not mergeable with a standard-profile one. + +--- + +## Section 3: Payload + +Per sketch. **Raw state only**, a **positional msgpack array** in the order its kind_id implies. Rules: + +- No field that `kind_id` or the metadata already determines (no variant tag, no precision, no counter type, no mode). +- No field derivable from another (no HLL `precision`; no CMS `l1`/`l2`, which are `sum(count)` / `sum(count^2)` and are recomputed on decode). +- msgpack array (positional), never a keyed map. The exact msgpack types are in "Wire encoding rules". + +> Note: derived summaries like CMS `l1`/`l2` and `sum_counts`/`sum2_counts` live in the **delta / error-accounting** format (proto `CountMinState`), a separate wire format. +> They do not belong in the self-contained sketch payload. + +### 3.1: HLL payload (`0x01 0x01` / `0x01 0x02` / `0x01 0x03`) + +The variant is in `kind_id`, precision is in the metadata (and equals `log2(register count)`), so the only real state is the register bytes (plus three running scalars for HIP). + +**Classic / Ertl-MLE** (`0x01 0x01`, `0x01 0x02`), identical layout: + +| Pos | Field | Type | Notes | +| ----- | ------- | ------ | ------- | +| 0 | `registers` | bin | one byte per register; length is `2^precision` | + +**HIP** (`0x01 0x03`): + +| Pos | Field | Type | Notes | +| ----- | ------- | ------ | ------- | +| 0 | `registers` | bin | one byte per register | +| 1 | `hip_kxq0` | f64 | HIP running estimate state | +| 2 | `hip_kxq1` | f64 | | +| 3 | `hip_est` | f64 | | + +### 3.2: Count-Min payload (`0x02 0x00`) + +The `CountMin` struct is generic in memory (counter `i32`/`i64`/`i128`/`f64`, `RegularPath`/`FastPath`, Nitro, and so on). +**That freedom is kept in memory; nothing is forbidden.** +The wire supports a fixed set. The two parameters that shape it, **counter type** (`"i64"`/`"f64"`) and **mode** (`"fast"`/`"regular"`), live in the metadata, so the payload itself is just shape and counters: + +| Pos | Field | Type | Notes | +| ----- | ------- | ------ | ------- | +| 0 | `counts` | array | packed **row-major**, `rows*cols` cells; element type = `counter_type` | + +`rows` and `cols` live in the metadata as structural params (Section 2), so the payload omits them. +The payload is a **1-element positional array `[counts]`**, mirroring HLL Classic's `[registers]`. + +Wire counter types are `i64` and `f64` only (`i32` widens to `i64`; `i128` and exotic counters are not wire types). +`mode` records `RegularPath` vs `FastPath` because they place a key in different columns (compare `cm_regular_path_correctness` vs `cm_fast_path_correctness`), so a reader must know which to reproduce a query. +A counter type other than i64/f64, or non-`Vector2D` storage, must be converted first; see Section 5, "Converting an exotic in-memory sketch". +Both modes, `FastPath` and `RegularPath`, serialize directly (you'd only "convert" a mode to *change* it, which needs re-inserting the data). + +### 3.3 onward: payloads not yet designed + +The remaining `kind_id`s reserve a family byte with payload TBD (Section 1 registry has their "assigned in Go" status). Likely shape when designed: + +| kind_id | Sketch | Likely payload | +| --------- | -------- | --------- | +| `0x03 0x00` | Count-Min-with-heap (CMSHeap) | similar to current CMS | +| `0x04 0x00` | Count Sketch | similar to current CMS | +| `0x05 0x00` | DDSketch | straightforward bucket | +| `0x06 0x00` | KLL | next on the list; no CDF serialization | +| `0x07 0x00` | Hydra-KLL | same challenge as KLL | +| `0x08 0x00` | SetAggregator | aggregation envelope, distinct from a stand-alone sketch (Section 1 mapping notes) | +| `0x09 0x00` | DeltaResult | delta-result envelope, distinct from a stand-alone sketch (Section 1 mapping notes) | +| `0x0a 0x00` | Count-Sketch-with-heap (CSHeap) | similar to current CMS | +| `0x0b 0x00` | Elastic (`Unstable`) | "key" needs thought, else similar to CMS; sketch needs optimization first, worth one combined PR | +| `0x0c 0x00` | Coco (`Unstable`) | "key" needs thought, else similar to CMS | +| `0x0d 0x00` | UniformSampling (`Unstable`) | straightforward | +| `0x0e 0x00` | KMV (`Unstable`) | straightforward | +| `0x0f 0x00` | HashSketchEnsemble | may not need serialization | +| `0x10 0x00` | UnivMon | more complex but doable | +| `0x11 0x00` | UnivMon Optimized | TBD | +| `0x12 0x00` | NitroBatch | may want another serialization abstraction in Storage | +| `0x13 0x00` | ExponentialHistogram | needs to re-use other sketches' serialization | +| `0x14 0x00` | EHSketchList | TBD | +| `0x15 0x00` | EHUnivOptimized (`Unstable`) | TBD | +| `0x16 0x00` | OctoSketch | TBD | + +--- + +## Section 4: Wire encoding rules (byte-level) + +This is what makes two languages emit **identical bytes**. +msgpack fixes endianness and float format; these rules fix the family/width choices that libraries otherwise make differently. + +**Integer family + width rule (applies to every integer below).** +This is the single biggest cross-language trap: some Go msgpack libraries emit a *signed* `int` family for a positive `int64` while Rust's `rmp_serde` narrows it to the `uint` family. Pin it: + +- A **non-negative** integer is encoded in the msgpack **uint** family, at the **minimal width** for its value (e.g. `300` gives `cd 01 2c`, uint16; `1` gives positive fixint `01`). +- A **negative** integer is encoded in the msgpack **int** family, minimal width. +- `f64` is always full **float64** (`0xcb`), never narrowed to float32. + +The Go side MUST configure its encoder to match (uint-narrowing on, minimal width). +Golden byte-vectors lock it. + +**Metadata (msgpack map)** + +- Keys are the exact ASCII strings in Section 2. +- **Canonical key order** = the order fields are listed in Section 2 (hash-spec group, then structural-params group). Encoders MUST write in this order. (Order is irrelevant to decoding but required for byte-identical output.) +- Decoders reject **unknown keys** (Rust uses `#[serde(deny_unknown_fields)]`); v1 carries exactly the fixed field set (its values are the hasher's `HashProfile`: the standard profile or a custom one). +- Values: strings as msgpack `str`; `seed_list` as a msgpack array of integers (each per the family/width rule); all other integers per the family/width rule. + +**Payload (msgpack array)** + +- A msgpack **array**, elements in the Section 3 position order. +- `registers`: msgpack `bin` (one byte per register; matches Go's `[]byte`). +- `counts`: msgpack array; each element is an integer (per the family/width rule) when `counter_type == "i64"`, a **float64** when `"f64"`. +- HLL HIP `hip_*`: **float64**. + +(Count-Min `rows` / `cols` are carried as metadata integers per the family/width rule; see Section 2.) + +--- + +## Section 5: Implementation detail + +Guidance for future development; this is outside the byte contract. It covers how a Rust decoder validates the bytes, and how to bring an in-memory config the wire doesn't cover directly onto the wire. + +### Validation (decode side) + +Fail **closed** on any mismatch: + +1. `kind_id` is in the registry. +2. Every hash-spec field matches the **target hasher's** `HashProfile`: decode compares the read metadata against `hll_metadata::` / `cms_metadata::` for the exact type being decoded into, so it does not merely accept the standard profile. Bytes carrying a different profile are rejected. +3. Structural params are consistent with `kind_id` and the payload: + - HLL: `registers.len() == 2^precision ==` the target storage's register count. + - Count-Min: `counts` element type matches `counter_type`; `counts.len() == rows*cols`. + +### Converting an exotic in-memory sketch to a wire form + +The library provides no free wire serialization for exotic counters; only the owner knows if the mapping is lossless. +Convert to a canonical counter type, then serialize. +Doable **today** with existing public API (the pattern `SketchlibCms` already uses): + +```rust +// e.g. a u64-counter FastPath CMS to the i64 wire form +let (rows, cols) = (src.rows(), src.cols()); +let converted: CountMin, FastPath> = CountMin::from_storage( + Vector2D::from_fn(rows, cols, |r, c| src.as_storage().query_one_counter(r, c) as i64), +); +let bytes = converted.serialize_to_bytes()?; // wire-eligible type +``` + +Converts the **counter type** only (cell-for-cell). +It does not convert the mode (Regular to/from Fast); that would need re-inserting the original data. + +--- + +## Cross-language contract + +Direction: **custom per-sketch payload replaces the `portable` types, and `sketchlib-go` mirrors each payload.** +Good direction (more compact, higher fidelity, less Rust-internal duplication), but it moves the contract from shared code to discipline. To keep it safe: + +1. **This spec**: byte-level, language-neutral, per sketch. +2. **Golden byte-vector fixtures** checked into both repos; both languages decode and re-encode them byte-identically. These replace the `portable`-as-oracle round-trip test that guards drift today. +3. **This registry**, mirrored, never independently allocated. + +**Hash profile on the Go side.** +Rust derives the hash spec from a generic `HashProfile` bound on the hasher type; Go has no generic hasher type, so there is nothing to derive from. +On the Go side the profile is simply **written into** the metadata on encode and **read from** it on decode. +Go MUST validate the profile it reads (same fail-closed intent as Rust): a sketch is only mergeable/queryable if its `hash_profile_id` + seeds match the profile Go is prepared to reproduce. + +Sequencing: do not delete `portable` until (2) exists; the current `native bytes == portable bytes` test is the only drift guard right now. +Keep it through the transition, retire `portable` once goldens are in place. + +--- + +## Decisions (resolved) + +- **kind_id = algorithm identity** (parameters live in the metadata). Structural params (HLL precision, CMS counter type + mode) live in metadata, which is read before the payload. Payload structure = kind_id + metadata. +- **Q-META**: metadata is a msgpack **map**; canonical key order per Section 4; optional fields are omitted keys. +- **Q-SEEDS**: `seed_list` is **inlined** in v1 so the bytes self-describe the hash. Resolving seeds from `hash_profile_id` alone is a v2 space optimization. Each sketch still carries only the seed *index* it uses. +- **Q-PROFILE**: the hash-spec metadata is **derived from the hasher's `HashProfile`** (`hll_metadata::` / `cms_metadata::`) and never hardcoded, so it is always truthful to the hasher. Custom hash profiles are **supported and self-describing**. Merge compatibility is hash-spec equality, so a custom-profile sketch is not mergeable with a standard one. +- **Q-CMS**: Count-Min is one `kind_id` (`0x02 0x00`); counter type and mode live in the metadata, so the id stays single. +- **Q-CMS-DIMS**: Count-Min `rows`/`cols` are **metadata** and the payload omits them. They are configuration that shapes the payload (like HLL's `precision`), so per the config-to-metadata rule they belong in the descriptor. The payload is then just `[counts]`. Canonical structural-param order: `... matrix_seed_index, rows, cols, counter_type, mode`. +- **Q-VER**: no payload version field. A new incompatible encoding gets a **new `kind_id`**; retired ids are reserved forever and never recycled. +- **Encoding**: metadata + payload are both msgpack; payload is a positional array. Byte-level rules in Section 4. diff --git a/examples/serialize_deserialize.rs b/examples/serialize_deserialize.rs index ba36a3c..bdaece5 100644 --- a/examples/serialize_deserialize.rs +++ b/examples/serialize_deserialize.rs @@ -11,14 +11,14 @@ use asap_sketchlib::{CountMin, DataInput, ErtlMLE, HyperLogLog, KLL, RegularPath fn main() { // --- Count-Min Sketch --- - let mut cms: CountMin, RegularPath> = CountMin::with_dimensions(5, 1024); + let mut cms: CountMin, RegularPath> = CountMin::with_dimensions(5, 1024); for id in 0u64..1_000 { cms.insert(&DataInput::U64(id % 100)); // 100 distinct IDs } let before = cms.estimate(&DataInput::U64(42)); let bytes = cms.serialize_to_bytes().expect("CMS serialize"); - let cms2 = CountMin::, RegularPath>::deserialize_from_bytes(&bytes) + let cms2 = CountMin::, RegularPath>::deserialize_from_bytes(&bytes) .expect("CMS deserialize"); let after = cms2.estimate(&DataInput::U64(42)); diff --git a/src/common/hash.rs b/src/common/hash.rs index ad8cfbc..355cdfd 100644 --- a/src/common/hash.rs +++ b/src/common/hash.rs @@ -67,11 +67,50 @@ pub trait SketchHasher: Clone + Debug { ) -> Self::HashType; } +/// Describes, in ASAPv1 wire terms, *how* a [`SketchHasher`] hashes — the hash +/// spec that a serialized sketch carries in its metadata (see +/// `docs/asapv1_wire_format.md` §2). Serialization is bounded on this trait so +/// the metadata is **derived from the hasher** rather than hardcoded: an +/// unprofiled hasher simply cannot be serialized (a compile-time guarantee that +/// fails closed), and a custom hasher that declares a profile serializes +/// truthfully — because `seed_list()` is inlined, its bytes are fully +/// self-describing on the wire. +pub trait HashProfile { + /// Stable global id, e.g. `"projectasap.xxh3.seedlist.v1"` (authoritative). + const PROFILE_ID: &'static str; + /// Hash algorithm identifier, e.g. `"xxh3_64_128"`. + const ALGORITHM: &'static str; + /// Seed-derivation scheme, e.g. `"seed_list_index_wrap"`. + const SEED_DERIVATION: &'static str; + /// Input-encoding identifier, e.g. `"projectasap.input.v1"`. + const INPUT_ENCODING: &'static str; + /// The full seed list, inlined into the metadata so the bytes self-describe. + fn seed_list() -> Vec; + /// Seed-list index a single-hash sketch (HLL) hashes with. + const CANONICAL_SEED_INDEX: u32; + /// Seed-list index a matrix-backed sketch (Count-Min) hashes rows with. + const MATRIX_SEED_INDEX: u32; +} + /// Default hasher using twox_hash (XxHash3). This is the built-in implementation /// used when no custom hasher is specified. #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct DefaultXxHasher; +/// The standard ProjectASAP hash profile (`docs/asapv1_wire_format.md` §2). +/// This is the single source of truth for the profile's wire values. +impl HashProfile for DefaultXxHasher { + const PROFILE_ID: &'static str = "projectasap.xxh3.seedlist.v1"; + const ALGORITHM: &'static str = "xxh3_64_128"; + const SEED_DERIVATION: &'static str = "seed_list_index_wrap"; + const INPUT_ENCODING: &'static str = "projectasap.input.v1"; + fn seed_list() -> Vec { + SEEDLIST.to_vec() + } + const CANONICAL_SEED_INDEX: u32 = CANONICAL_HASH_SEED as u32; + const MATRIX_SEED_INDEX: u32 = 0; +} + impl SketchHasher for DefaultXxHasher { type HashType = MatrixHashType; diff --git a/src/common/input.rs b/src/common/input.rs index 0fbd337..2eb3e19 100644 --- a/src/common/input.rs +++ b/src/common/input.rs @@ -483,7 +483,7 @@ impl HydraCounter { (HydraCounter::UNIVERSAL(um), HydraQuery::L1Norm) => Ok(um.calc_l1()), (HydraCounter::UNIVERSAL(um), HydraQuery::L2Norm) => Ok(um.calc_l2()), (HydraCounter::UNIVERSAL(um), HydraQuery::Entropy) => Ok(um.calc_entropy()), - (c, q) => Err(format!("{} does not support {}", c, q)), + (c, q) => Err(format!("{c} does not support {q}")), } } diff --git a/src/common/mod.rs b/src/common/mod.rs index 87882d4..c8edb7d 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -34,9 +34,9 @@ pub mod structure_utils; pub mod structures; pub use hash::{ - BOTTOM_LAYER_FINDER, CANONICAL_HASH_SEED, DefaultXxHasher, HYDRA_SEED, MatrixHashMode, - SEEDLIST, SketchHasher, hash_for_matrix, hash_for_matrix_generic, hash_for_matrix_seeded, - hash_for_matrix_seeded_generic, hash_for_matrix_seeded_with_mode, + BOTTOM_LAYER_FINDER, CANONICAL_HASH_SEED, DefaultXxHasher, HYDRA_SEED, HashProfile, + MatrixHashMode, SEEDLIST, SketchHasher, hash_for_matrix, hash_for_matrix_generic, + hash_for_matrix_seeded, hash_for_matrix_seeded_generic, hash_for_matrix_seeded_with_mode, hash_for_matrix_seeded_with_mode_generic, hash_item64_seeded, hash_item128_seeded, hash_mode_for_matrix, hash64_seeded, hash128_seeded, }; diff --git a/src/common/structure_utils.rs b/src/common/structure_utils.rs index 29e9165..a17e461 100644 --- a/src/common/structure_utils.rs +++ b/src/common/structure_utils.rs @@ -365,8 +365,7 @@ mod tests { let sort_median = median_three_sort(v); assert_eq!( fast_median, sort_median, - "median for sort is {sort_median} but fast gives {fast_median}, input is {:?}", - v + "median for sort is {sort_median} but fast gives {fast_median}, input is {v:?}" ); } for v in &mut four_vec { @@ -374,8 +373,7 @@ mod tests { let sort_median = median_four_sort(v); assert_eq!( fast_median, sort_median, - "median for sort is {sort_median} but fast gives {fast_median}, input is {:?}", - v + "median for sort is {sort_median} but fast gives {fast_median}, input is {v:?}" ); } for v in &mut five_vec { @@ -383,8 +381,7 @@ mod tests { let sort_median = median_five_sort(v); assert_eq!( fast_median, sort_median, - "median for sort is {sort_median} but fast gives {fast_median}, input is {:?}", - v + "median for sort is {sort_median} but fast gives {fast_median}, input is {v:?}" ); } } diff --git a/src/message_pack_format/envelope.rs b/src/message_pack_format/envelope.rs new file mode 100644 index 0000000..2c5826c --- /dev/null +++ b/src/message_pack_format/envelope.rs @@ -0,0 +1,150 @@ +//! Shared ASAPv1 envelope framing (sketch-agnostic). +//! +//! Every serialized ASAP sketch binary is wrapped in this envelope: +//! +//! ```text +//! [ magic:6 | version:u8 | kind_id_len:u8 | kind_id:bytes +//! | metadata_len:u32_be | payload_len:u32_be +//! | metadata:msgpack | payload:msgpack ] +//! ``` +//! +//! This module owns only the parts that are identical for every sketch: the +//! magic sentinel, the layout version, and the byte framing ([`encode`] / +//! [`split`]). The hash-profile values each sketch's metadata is built from live +//! with the hasher itself (see [`crate::HashProfile`]). The `kind_id`, the +//! metadata *contents*, and the payload are per-sketch and defined alongside +//! each sketch — see `docs/asapv1_wire_format.md`. +//! +//! [`split`] validates only the magic, version, and framing; it does **not** +//! check `kind_id` against a registry. Each sketch decoder checks that the +//! `kind_id` is one it owns. + +/// 6-byte ASCII sentinel opening every ASAP sketch binary (`b"ASAPv1"`). +pub(crate) const MAGIC: &[u8; 6] = b"ASAPv1"; + +/// Envelope layout version. Bumped only when the framing itself changes. +pub(crate) const VERSION: u8 = 0x01; + +/// Assemble the envelope around an already-encoded metadata block and payload. +pub(crate) fn encode(kind_id: &[u8], metadata: &[u8], payload: &[u8]) -> Vec { + let kind_id_len = u8::try_from(kind_id.len()).expect("ASAPv1 kind_id too long (>255 bytes)"); + let metadata_len = u32::try_from(metadata.len()).expect("ASAPv1 metadata too large"); + let payload_len = u32::try_from(payload.len()).expect("ASAPv1 payload too large"); + let mut out = Vec::with_capacity( + MAGIC.len() + 1 + 1 + kind_id.len() + 4 + 4 + metadata.len() + payload.len(), + ); + out.extend_from_slice(MAGIC); + out.push(VERSION); + out.push(kind_id_len); + out.extend_from_slice(kind_id); + out.extend_from_slice(&metadata_len.to_be_bytes()); + out.extend_from_slice(&payload_len.to_be_bytes()); + out.extend_from_slice(metadata); + out.extend_from_slice(payload); + out +} + +/// The three envelope slices returned by [`split`]: `(kind_id, metadata, payload)`. +pub(crate) type Parts<'a> = (&'a [u8], &'a [u8], &'a [u8]); + +/// Split an envelope into `(kind_id, metadata, payload)`, validating the magic, +/// version, and that the two length-framed blocks fit. Returns an error string +/// on any structural mismatch. Does **not** validate `kind_id` — the caller does. +pub(crate) fn split(bytes: &[u8]) -> Result, String> { + let magic_len = MAGIC.len(); + let kind_id_len_offset = magic_len + 1; + let kind_id_offset = magic_len + 2; + let header_min = kind_id_offset + 4 + 4; + + if bytes.len() < header_min { + return Err(format!( + "ASAPv1 envelope: too short ({} bytes, need at least {header_min})", + bytes.len() + )); + } + if &bytes[..magic_len] != MAGIC { + return Err("ASAPv1 envelope: bad magic".to_string()); + } + if bytes[magic_len] != VERSION { + return Err(format!( + "ASAPv1 envelope: unsupported version 0x{:02x}", + bytes[magic_len] + )); + } + let kind_id_len = bytes[kind_id_len_offset] as usize; + let lengths_offset = kind_id_offset + kind_id_len; + if bytes.len() < lengths_offset + 8 { + return Err("ASAPv1 envelope: truncated kind_id / length fields".to_string()); + } + let kind_id = &bytes[kind_id_offset..lengths_offset]; + + let metadata_len = u32::from_be_bytes( + bytes[lengths_offset..lengths_offset + 4] + .try_into() + .expect("four-byte length slice"), + ) as usize; + let payload_len = u32::from_be_bytes( + bytes[lengths_offset + 4..lengths_offset + 8] + .try_into() + .expect("four-byte length slice"), + ) as usize; + let metadata_start = lengths_offset + 8; + // Use checked arithmetic so crafted lengths near usize::MAX (reachable on + // 32-bit targets) fail closed with an error instead of overflowing the + // bounds check and panicking on the slice index. + let payload_start = metadata_start + .checked_add(metadata_len) + .ok_or_else(|| "ASAPv1 envelope: length overflow".to_string())?; + let payload_end = payload_start + .checked_add(payload_len) + .ok_or_else(|| "ASAPv1 envelope: length overflow".to_string())?; + if bytes.len() < payload_end { + return Err("ASAPv1 envelope: truncated metadata / payload".to_string()); + } + Ok(( + kind_id, + &bytes[metadata_start..payload_start], + &bytes[payload_start..payload_end], + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trip_frames_metadata_and_payload() { + let kind_id = [0x01u8, 0x03]; + let metadata = b"meta-block"; + let payload = b"payload-bytes"; + let bytes = encode(&kind_id, metadata, payload); + + assert!(bytes.starts_with(MAGIC)); + assert_eq!(bytes[6], VERSION); + assert_eq!(bytes[7], 2); // kind_id_len + + let (k, m, p) = split(&bytes).expect("split"); + assert_eq!(k, kind_id); + assert_eq!(m, metadata); + assert_eq!(p, payload); + } + + #[test] + fn rejects_bad_magic_and_version() { + let mut bytes = encode(&[0x01, 0x01], b"m", b"p"); + assert!(split(&bytes).is_ok()); + bytes[0] = b'X'; + assert!(split(&bytes).is_err()); + + let mut bytes = encode(&[0x01, 0x01], b"m", b"p"); + bytes[6] = 0xFF; // version + assert!(split(&bytes).is_err()); + } + + #[test] + fn rejects_truncation() { + let bytes = encode(&[0x01, 0x01], b"metadata", b"payload"); + assert!(split(&bytes[..bytes.len() - 1]).is_err()); + assert!(split(&bytes[..3]).is_err()); + } +} diff --git a/src/message_pack_format/mod.rs b/src/message_pack_format/mod.rs index 7734463..6b3c3f0 100644 --- a/src/message_pack_format/mod.rs +++ b/src/message_pack_format/mod.rs @@ -19,6 +19,7 @@ //! top level so both worlds share the same encode/decode contract. pub mod codec; +pub(crate) mod envelope; pub mod error; pub mod native; pub mod portable; diff --git a/src/message_pack_format/native/countminsketch.rs b/src/message_pack_format/native/countminsketch.rs index bed39e2..ee90c29 100644 --- a/src/message_pack_format/native/countminsketch.rs +++ b/src/message_pack_format/native/countminsketch.rs @@ -1,14 +1,21 @@ //! Native MessagePack codec impl for [`crate::sketches::countminsketch::CountMin`]. +//! +//! Only the canonical wire configs — i64/f64 counters (`CmsWireCounter`) with a +//! fast/regular mode (`CmsWireMode`) — are serializable. Exotic in-memory +//! counters (i32/i128/…) must be converted to a wire type first. use serde::{Deserialize, Serialize}; use crate::message_pack_format::{Error, MessagePackCodec}; -use crate::sketches::countminsketch::CountMin; -use crate::{MatrixStorage, SketchHasher}; +use crate::sketches::countminsketch::{CmsWireCounter, CmsWireMode, CountMin}; +use crate::{HashProfile, SketchHasher, Vector2D}; -impl MessagePackCodec for CountMin +impl MessagePackCodec for CountMin, Mode, H> where - S: MatrixStorage + Serialize + for<'de> Deserialize<'de>, + // `AddAssign` is required for `Vector2D: MatrixStorage`. + T: CmsWireCounter + std::ops::AddAssign + Serialize + for<'de> Deserialize<'de>, + Mode: CmsWireMode, + H: SketchHasher + HashProfile, { fn to_msgpack(&self) -> Result, Error> { Ok(self.serialize_to_bytes()?) diff --git a/src/message_pack_format/native/hll.rs b/src/message_pack_format/native/hll.rs index 36f76e9..b5de311 100644 --- a/src/message_pack_format/native/hll.rs +++ b/src/message_pack_format/native/hll.rs @@ -1,11 +1,11 @@ //! Native MessagePack codec impls for [`crate::sketches::hll`]. use crate::message_pack_format::{Error, MessagePackCodec}; -use crate::sketches::hll::{HyperLogLogHIPImpl, HyperLogLogImpl}; -use crate::{HllRegisterStorage, SketchHasher}; +use crate::sketches::hll::{HllWireVariant, HyperLogLogHIPImpl, HyperLogLogImpl}; +use crate::{HashProfile, HllRegisterStorage, SketchHasher}; -impl MessagePackCodec - for HyperLogLogImpl +impl + MessagePackCodec for HyperLogLogImpl { fn to_msgpack(&self) -> Result, Error> { Ok(self.serialize_to_bytes()?) diff --git a/src/message_pack_format/portable/countminsketch_topk.rs b/src/message_pack_format/portable/countminsketch_topk.rs index fa850f9..0227824 100644 --- a/src/message_pack_format/portable/countminsketch_topk.rs +++ b/src/message_pack_format/portable/countminsketch_topk.rs @@ -86,7 +86,7 @@ pub fn heap_to_wire(cms_heap: &SketchlibCMSHeap) -> Vec { .map(|hh_item| { let key = match &hh_item.key { crate::HeapItem::String(s) => s.clone(), - other => format!("{:?}", other), + other => format!("{other:?}"), }; WireHeapItem { key, diff --git a/src/message_pack_format/portable/ddsketch.rs b/src/message_pack_format/portable/ddsketch.rs index 09e0369..2fce408 100644 --- a/src/message_pack_format/portable/ddsketch.rs +++ b/src/message_pack_format/portable/ddsketch.rs @@ -702,13 +702,11 @@ mod tests { // P99 ≈ exp(mu + sigma * Φ⁻¹(0.99)) = e^(3 + 0.7×2.326) ≈ 102.4. assert!( (p50 / 20.09).ln().abs() < 0.05, - "P50 {} not close to 20.09", - p50 + "P50 {p50} not close to 20.09" ); assert!( (p99 / 102.4).ln().abs() < 0.05, - "P99 {} not close to 102.4", - p99 + "P99 {p99} not close to 102.4" ); } @@ -782,12 +780,7 @@ mod tests { let rel_err = (got / want - 1.0).abs(); assert!( rel_err <= alpha, - "q={} rel_err={:.4} exceeds α={}: reconstituted={}, full={}", - q, - rel_err, - alpha, - got, - want, + "q={q} rel_err={rel_err:.4} exceeds α={alpha}: reconstituted={got}, full={want}", ); } } diff --git a/src/message_pack_format/portable/hll.rs b/src/message_pack_format/portable/hll.rs index f744566..482ed97 100644 --- a/src/message_pack_format/portable/hll.rs +++ b/src/message_pack_format/portable/hll.rs @@ -11,6 +11,44 @@ use serde::{Deserialize, Serialize}; use crate::message_pack_format::{Error as MsgPackError, MessagePackCodec}; use crate::{CANONICAL_HASH_SEED, DataInput, hash64_seeded}; +// --------------------------------------------------------------------------- +// DEPRECATED — temporary compatibility shim, slated for removal. +// +// A few call sites still depend on this portable `HllSketch`. Its ASAPv1 codec +// now reuses the shared `envelope` framing plus the HLL metadata/payload types +// in `crate::sketches::hll`, so it stays byte-identical with the native +// `HyperLogLogImpl` / `HyperLogLogHIPImpl` serialization. Do NOT add new callers. +// +// Prefer instead the native path: +// `sketches::hll::HyperLogLogImpl::serialize_to_bytes` / +// `HyperLogLogHIPImpl::serialize_to_bytes` (+ `deserialize_from_bytes`). +// --------------------------------------------------------------------------- + +use crate::message_pack_format::envelope; +use crate::sketches::hll::{ + HLL_KIND_CLASSIC, HLL_KIND_ERTL_MLE, HLL_KIND_HIP, HllMetadata, HllPayloadHip, HllPayloadPlain, + standard_hll_metadata, +}; + +/// Retained only for `test_msgpack_round_trip`'s magic-prefix assertion. +#[cfg(test)] +const HLL_WRAPPER_MAGIC: &[u8; 6] = b"ASAPv1"; + +/// Map a portable [`HllVariant`] to its ASAPv1 wire `kind_id`. `Unspecified` is +/// a placeholder with no wire form. +fn wire_kind_id(variant: HllVariant) -> Result<&'static [u8], MsgPackError> { + match variant { + HllVariant::Regular => Ok(HLL_KIND_CLASSIC), + HllVariant::Datafusion => Ok(HLL_KIND_ERTL_MLE), + HllVariant::Hip => Ok(HLL_KIND_HIP), + HllVariant::Unspecified => Err(MsgPackError::Decode( + rmp_serde::decode::Error::Uncategorized( + "ASAPv1 HLL: the Unspecified variant is not serializable".to_string(), + ), + )), + } +} + /// HLL estimator variant. Mirrors `asap_sketchlib::proto::sketchlib::HllVariant`. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum HllVariant { @@ -475,11 +513,87 @@ fn read_uvarint(buf: &[u8]) -> Option<(u64, usize)> { impl MessagePackCodec for HllSketch { fn to_msgpack(&self) -> Result, MsgPackError> { - Ok(rmp_serde::to_vec(self)?) + let kind_id = wire_kind_id(self.variant)?; + let metadata = rmp_serde::to_vec_named(&standard_hll_metadata(self.precision))?; + let payload = if self.variant == HllVariant::Hip { + rmp_serde::to_vec(&HllPayloadHip { + registers: self.registers.clone(), + hip_kxq0: self.hip_kxq0, + hip_kxq1: self.hip_kxq1, + hip_est: self.hip_est, + })? + } else { + rmp_serde::to_vec(&HllPayloadPlain { + registers: self.registers.clone(), + })? + }; + Ok(envelope::encode(kind_id, &metadata, &payload)) } fn from_msgpack(bytes: &[u8]) -> Result { - Ok(rmp_serde::from_slice(bytes)?) + let (kind_id, metadata, payload) = envelope::split(bytes) + .map_err(|msg| MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(msg)))?; + + // Validate the hash spec is the standard profile (precision is read from + // the metadata, so compare against a standard block built with it). + let meta: HllMetadata = rmp_serde::from_slice(metadata)?; + if meta != standard_hll_metadata(meta.precision) { + return Err(MsgPackError::Decode( + rmp_serde::decode::Error::Uncategorized( + "ASAPv1 HLL envelope: hash metadata mismatch".to_string(), + ), + )); + } + + let variant = if kind_id == HLL_KIND_CLASSIC { + HllVariant::Regular + } else if kind_id == HLL_KIND_ERTL_MLE { + HllVariant::Datafusion + } else if kind_id == HLL_KIND_HIP { + HllVariant::Hip + } else { + return Err(MsgPackError::Decode( + rmp_serde::decode::Error::Uncategorized(format!( + "ASAPv1 HLL envelope: unsupported kind_id {kind_id:?}" + )), + )); + }; + + let (registers, hip_kxq0, hip_kxq1, hip_est) = if variant == HllVariant::Hip { + let p: HllPayloadHip = rmp_serde::from_slice(payload)?; + (p.registers, p.hip_kxq0, p.hip_kxq1, p.hip_est) + } else { + let p: HllPayloadPlain = rmp_serde::from_slice(payload)?; + (p.registers, 0.0, 0.0, 0.0) + }; + + // Fail closed if the register bin does not match `2^precision` + // (doc §2 validation rule 3). `checked_shl` avoids a shift-overflow + // panic on a crafted out-of-range `precision`. + let expected = 1usize.checked_shl(meta.precision).ok_or_else(|| { + MsgPackError::Decode(rmp_serde::decode::Error::Uncategorized(format!( + "ASAPv1 HLL envelope: precision {} out of range", + meta.precision + ))) + })?; + if registers.len() != expected { + return Err(MsgPackError::Decode( + rmp_serde::decode::Error::Uncategorized(format!( + "ASAPv1 HLL envelope: {} registers, expected 2^{} = {expected}", + registers.len(), + meta.precision + )), + )); + } + + Ok(HllSketch::from_raw( + variant, + meta.precision, + registers, + hip_kxq0, + hip_kxq1, + hip_est, + )) } } @@ -655,6 +769,7 @@ mod tests { 3.0, ); let bytes = original.to_msgpack().unwrap(); + assert!(bytes.starts_with(HLL_WRAPPER_MAGIC)); let decoded = HllSketch::from_msgpack(&bytes).unwrap(); assert_eq!(decoded.registers, original.registers); assert_eq!(decoded.precision, original.precision); diff --git a/src/sketch_framework/hydra.rs b/src/sketch_framework/hydra.rs index 31f61aa..48612d4 100644 --- a/src/sketch_framework/hydra.rs +++ b/src/sketch_framework/hydra.rs @@ -291,8 +291,7 @@ impl MultiHeadHydra { } if std::mem::discriminant(counter) != std::mem::discriminant(other_counter) { return Err(format!( - "MultiHeadHydra counter type mismatch for dimension '{}'", - name + "MultiHeadHydra counter type mismatch for dimension '{name}'" )); } } @@ -715,8 +714,7 @@ mod tests { let quantile = hydra.query_key(vec!["metrics", "latency"], &HydraQuery::Cdf(30.0)); assert!( (quantile - 0.6).abs() < 1e-9, - "expected CDF near 0.6, got {}", - quantile + "expected CDF near 0.6, got {quantile}" ); let empty_bucket = hydra.query_key(vec!["other", "key"], &HydraQuery::Cdf(50.0)); @@ -835,8 +833,7 @@ mod tests { let card = result.unwrap(); assert!( card > 90.0 && card < 110.0, - "Expected approx 100, got {}", - card + "Expected approx 100, got {card}" ); } @@ -858,8 +855,7 @@ mod tests { let median = result.unwrap(); assert!( (median - 50.0).abs() < 5.0, - "Expected approx 50, got {}", - median + "Expected approx 50, got {median}" ); } diff --git a/src/sketch_framework/univmon.rs b/src/sketch_framework/univmon.rs index 9d22f23..8fee9ba 100644 --- a/src/sketch_framework/univmon.rs +++ b/src/sketch_framework/univmon.rs @@ -146,7 +146,7 @@ impl UnivMon { pub fn print_hh_layer(&self) { print!("Print HH_Layer: "); for i in 0..self.layer_size { - println!("layer {}: ", i); + println!("layer {i}: "); self.hh_layers[i].print_heap(); } } @@ -635,7 +635,7 @@ mod tests { for (prefix, count, repeat) in scenarios { for i in 0..repeat { - let key = format!("{}_{}", prefix, i); + let key = format!("{prefix}_{i}"); let val = count as i64; let val_f = val as f64; diff --git a/src/sketches/countminsketch.rs b/src/sketches/countminsketch.rs index 43f296f..9fa49d9 100644 --- a/src/sketches/countminsketch.rs +++ b/src/sketches/countminsketch.rs @@ -8,9 +8,6 @@ //! Sketch and its Applications," J. Algorithms 55(1), 2005. //! -use rmp_serde::{ - decode::Error as RmpDecodeError, encode::Error as RmpEncodeError, from_slice, to_vec_named, -}; use serde::{Deserialize, Serialize}; use std::marker::PhantomData; @@ -21,6 +18,9 @@ use crate::{ QuickMatrixI128, RegularPath, SketchHasher, Vector2D, hash64_seeded, }; +mod wire; +pub(crate) use wire::{CmsWireCounter, CmsWireMode}; + const DEFAULT_ROW_NUM: usize = 3; const DEFAULT_COL_NUM: usize = 4096; /// Recommended row count for quick-start examples. @@ -260,21 +260,6 @@ impl CountMin { } } -// Serialization helpers for CountMin. -impl CountMin { - /// Serializes the sketch into MessagePack bytes. - pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - to_vec_named(self) - } -} - -impl Deserialize<'de>, Mode, H: SketchHasher> CountMin { - /// Deserializes a sketch from MessagePack bytes. - pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - from_slice(bytes) - } -} - // DataInput adapters for the regular Count-Min update rule. // Regular-path CountMin operations. Uses PartialOrd to support both integer and f64 counters. impl CountMin @@ -938,25 +923,4 @@ mod tests { "in-bound items number {within_count} not greater than expected amount {correct_lower_bound}" ); } - - #[test] - fn count_min_round_trip_serialization() { - let mut sketch = CountMin::, RegularPath>::with_dimensions(3, 8); - sketch.insert(&DataInput::U64(42)); - sketch.insert(&DataInput::U64(7)); - - let encoded = sketch.serialize_to_bytes().expect("serialize CountMin"); - assert!(!encoded.is_empty()); - let data_copied = encoded.clone(); - - let decoded = CountMin::, RegularPath>::deserialize_from_bytes(&data_copied) - .expect("deserialize CountMin"); - - assert_eq!(sketch.rows(), decoded.rows()); - assert_eq!(sketch.cols(), decoded.cols()); - assert_eq!( - sketch.as_storage().as_slice(), - decoded.as_storage().as_slice() - ); - } } diff --git a/src/sketches/countminsketch/wire.rs b/src/sketches/countminsketch/wire.rs new file mode 100644 index 0000000..24b4e2e --- /dev/null +++ b/src/sketches/countminsketch/wire.rs @@ -0,0 +1,355 @@ +//! ASAPv1 wire serialization for the Count-Min sketch. +//! +//! Child submodule of [`crate::sketches::countminsketch`]: it holds ALL of +//! Count-Min's serialization (the metadata/payload DTOs, the kind_id constant, +//! the [`CmsWireCounter`] / [`CmsWireMode`] marker traits, and the +//! `serialize_to_bytes` / `deserialize_from_bytes` impls) while the algorithm +//! lives in the parent module file. Being a descendant module, it reads the +//! sketch's private `counts` field directly without widening any field +//! visibility. See `docs/asapv1_wire_format.md` §3.2. +//! +//! Count-Min is one algorithm — a single kind_id `0x02 0x00`. The structural +//! parameters — the matrix dimensions (`rows` / `cols`), the **counter type** +//! (i64/f64) and the column-derivation **mode** (fast/regular) — all live in the +//! metadata, so the payload itself is just `[counts]` (a 1-element array +//! mirroring HLL Classic's `[registers]`). Only the canonical wire configs +//! (i64/f64 counters × fast/regular) get a serialization; exotic in-memory +//! counters (i32/i128/…) must be converted to a wire type first. + +use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError, from_slice}; +use serde::{Deserialize, Serialize}; + +use crate::message_pack_format::envelope; +use crate::{FastPath, HashProfile, RegularPath, SketchHasher, Vector2D}; + +use super::CountMin; + +/// CMS kind_id: family `0x02`, single algorithm variant `0x00`. +const CMS_KIND: &[u8] = &[0x02, 0x00]; + +/// Names the wire counter type carried in the metadata (`counter_type`). +/// Implemented only for the two wire-eligible counter types. +pub trait CmsWireCounter: Copy { + /// Metadata `counter_type` string — `"i64"` or `"f64"`. + const COUNTER_TYPE: &'static str; +} +impl CmsWireCounter for i64 { + const COUNTER_TYPE: &'static str = "i64"; +} +impl CmsWireCounter for f64 { + const COUNTER_TYPE: &'static str = "f64"; +} + +/// Names the wire column-derivation mode carried in the metadata (`mode`). +pub trait CmsWireMode { + /// Metadata `mode` string — `"fast"` or `"regular"`. + const MODE: &'static str; +} +impl CmsWireMode for RegularPath { + const MODE: &'static str = "regular"; +} +impl CmsWireMode for FastPath { + const MODE: &'static str = "fast"; +} + +/// CMS descriptor metadata (ASAPv1 §2), a msgpack **map** (`to_vec_named`) with +/// keys in this declaration order — the canonical order the wire spec fixes +/// (Go must mirror it). Hash-spec fields first, then the structural params +/// `rows` / `cols` / `counter_type` / `mode`. Per the spec's config→metadata +/// rule, the matrix dimensions are configuration (like HLL's `precision`) and so +/// live here rather than in the payload. +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct CmsMetadata { + metadata_version: u8, + hash_profile_id: String, + hash_algorithm: String, + seed_derivation: String, + input_encoding: String, + seed_list: Vec, + matrix_seed_index: u32, + rows: u32, + cols: u32, + counter_type: String, + mode: String, +} + +/// Builds the CMS descriptor metadata from the hasher's [`HashProfile`], so the +/// wire bytes truthfully describe how the sketch was hashed (rather than +/// hardcoding the standard profile). `matrix_seed_index` is the profile's own +/// row seed index; `rows` / `cols` are the sketch's structural dimensions. +fn cms_metadata( + rows: u32, + cols: u32, + counter_type: &str, + mode: &str, +) -> CmsMetadata { + CmsMetadata { + metadata_version: 1, + hash_profile_id: H::PROFILE_ID.to_string(), + hash_algorithm: H::ALGORITHM.to_string(), + seed_derivation: H::SEED_DERIVATION.to_string(), + input_encoding: H::INPUT_ENCODING.to_string(), + seed_list: H::seed_list(), + matrix_seed_index: H::MATRIX_SEED_INDEX, + rows, + cols, + counter_type: counter_type.to_string(), + mode: mode.to_string(), + } +} + +/// CMS payload (ASAPv1 §3.2), a msgpack **array** (`to_vec`, positional): +/// `[counts]` — a 1-element array (mirroring HLL Classic's `[registers]`). The +/// dimensions live in the metadata; `counts` is packed row-major and its element +/// type is fixed by the metadata `counter_type`. +#[derive(Debug, Serialize, Deserialize)] +struct CmsPayload { + counts: Vec, +} + +// Wire serialization for the canonical Count-Min configs only. `wire` is a +// descendant of the sketch module, so this impl reads the private `counts` +// field directly. +impl CountMin, Mode, H> +where + // `AddAssign` is required for `Vector2D: MatrixStorage` (the struct's + // bound), not by the bodies below. + T: CmsWireCounter + std::ops::AddAssign + Serialize + for<'de> Deserialize<'de>, + Mode: CmsWireMode, + H: SketchHasher + HashProfile, +{ + /// Serializes the sketch into an ASAPv1 MessagePack envelope. The metadata is + /// derived from the hasher's [`HashProfile`], so it truthfully describes how + /// the sketch was hashed. + pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { + let rows = self.counts.rows(); + let cols = self.counts.cols(); + let metadata = rmp_serde::to_vec_named(&cms_metadata::( + rows as u32, + cols as u32, + T::COUNTER_TYPE, + Mode::MODE, + ))?; + let payload = rmp_serde::to_vec(&CmsPayload:: { + counts: self.counts.as_slice().to_vec(), + })?; + Ok(envelope::encode(CMS_KIND, &metadata, &payload)) + } + + /// Deserializes a sketch from an ASAPv1 MessagePack envelope. The matrix + /// dimensions are read from the (validated) metadata; the payload carries + /// only the counts. + pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { + let (kind_id, metadata, payload) = + envelope::split(bytes).map_err(RmpDecodeError::Uncategorized)?; + if kind_id != CMS_KIND { + return Err(RmpDecodeError::Uncategorized(format!( + "CMS kind_id mismatch: stored {kind_id:?}, expected {CMS_KIND:?}" + ))); + } + let meta: CmsMetadata = from_slice(metadata)?; + // Validate the hash spec + counter type + mode against this target; + // `rows`/`cols` are structural (the sketch is dynamically sized), so they + // are echoed back into the expected block rather than known a priori. + if meta != cms_metadata::(meta.rows, meta.cols, T::COUNTER_TYPE, Mode::MODE) { + return Err(RmpDecodeError::Uncategorized( + "ASAPv1 CMS envelope: metadata mismatch".to_string(), + )); + } + let (rows, cols) = (meta.rows as usize, meta.cols as usize); + let p: CmsPayload = from_slice(payload)?; + // Reject zero dimensions before building the matrix: `Vector2D::from_fn` + // derives its mask via `cols.ilog2()`, which panics on `cols == 0`. Fail + // closed with an error rather than panicking on crafted bytes. + if rows == 0 || cols == 0 { + return Err(RmpDecodeError::Uncategorized(format!( + "CMS dimensions must be non-zero: rows={rows}, cols={cols}" + ))); + } + if p.counts.len() != rows.saturating_mul(cols) { + return Err(RmpDecodeError::Uncategorized(format!( + "CMS counts length {} != rows*cols {}", + p.counts.len(), + rows.saturating_mul(cols) + ))); + } + let storage = Vector2D::from_fn(rows, cols, |r, c| p.counts[r * cols + c]); + Ok(CountMin::from_storage(storage)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{CANONICAL_HASH_SEED, DataInput, DefaultXxHasher}; + + #[test] + fn count_min_round_trip_serialization() { + // i64 counters are a wire-eligible config; the ASAPv1 envelope round-trips. + let mut sketch = CountMin::, RegularPath>::with_dimensions(3, 8); + sketch.insert(&DataInput::U64(42)); + sketch.insert(&DataInput::U64(7)); + + let encoded = sketch.serialize_to_bytes().expect("serialize CountMin"); + assert!(encoded.starts_with(b"ASAPv1")); + assert_eq!(&encoded[7..10], &[2u8, 0x02, 0x00]); // kind_id_len=2, kind_id=[0x02,0x00] + + let decoded = CountMin::, RegularPath>::deserialize_from_bytes(&encoded) + .expect("deserialize CountMin"); + + assert_eq!(sketch.rows(), decoded.rows()); + assert_eq!(sketch.cols(), decoded.cols()); + assert_eq!( + sketch.as_storage().as_slice(), + decoded.as_storage().as_slice() + ); + } + + // A test-only custom hasher: hashes exactly like `DefaultXxHasher` but + // declares a DIFFERENT `HashProfile`. CMS metadata is derived from the + // profile, so an `AltHasher` sketch serializes truthfully. (An *unprofiled* + // hasher cannot serialize at all — that is a compile-time guarantee, since + // the wire methods are bounded on `H: HashProfile`.) + #[derive(Clone, Debug)] + struct AltHasher; + + impl SketchHasher for AltHasher { + type HashType = ::HashType; + + fn hash64_seeded(d: usize, key: &DataInput) -> u64 { + DefaultXxHasher::hash64_seeded(d, key) + } + fn hash128_seeded(d: usize, key: &DataInput) -> u128 { + DefaultXxHasher::hash128_seeded(d, key) + } + fn hash_item64_seeded(d: usize, key: &crate::HeapItem) -> u64 { + DefaultXxHasher::hash_item64_seeded(d, key) + } + fn hash_item128_seeded(d: usize, key: &crate::HeapItem) -> u128 { + DefaultXxHasher::hash_item128_seeded(d, key) + } + fn hash_for_matrix_seeded( + seed_idx: usize, + rows: usize, + cols: usize, + key: &DataInput, + ) -> Self::HashType { + DefaultXxHasher::hash_for_matrix_seeded(seed_idx, rows, cols, key) + } + } + + impl HashProfile for AltHasher { + const PROFILE_ID: &'static str = "test.alt.profile.v1"; + const ALGORITHM: &'static str = "xxh3_64_128"; + const SEED_DERIVATION: &'static str = "seed_list_index_wrap"; + const INPUT_ENCODING: &'static str = "projectasap.input.v1"; + fn seed_list() -> Vec { + vec![1, 2, 3, 4, 5] + } + const CANONICAL_SEED_INDEX: u32 = CANONICAL_HASH_SEED as u32; + const MATRIX_SEED_INDEX: u32 = 0; + } + + #[test] + fn count_min_custom_hasher_profile_round_trips_and_is_self_describing() { + // (a) A CMS built with a custom-profile hasher round-trips. + let mut alt = CountMin::, RegularPath, AltHasher>::with_dimensions(3, 8); + let mut std = CountMin::, RegularPath>::with_dimensions(3, 8); + alt.insert(&DataInput::U64(42)); + alt.insert(&DataInput::U64(7)); + std.insert(&DataInput::U64(42)); + std.insert(&DataInput::U64(7)); + + let alt_bytes = alt.serialize_to_bytes().expect("alt serialize"); + let decoded = + CountMin::, RegularPath, AltHasher>::deserialize_from_bytes(&alt_bytes) + .expect("alt decode"); + assert_eq!(alt.as_storage().as_slice(), decoded.as_storage().as_slice()); + + // (b) Bytes differ from the standard-profile sketch (metadata derived + // from the different profile). + let std_bytes = std.serialize_to_bytes().expect("std serialize"); + assert_ne!(alt_bytes, std_bytes); + + // (c) Standard-profile decode fails closed on custom-profile bytes. + assert!( + CountMin::, RegularPath>::deserialize_from_bytes(&alt_bytes).is_err(), + "standard-profile decode must reject custom-profile bytes" + ); + } + + #[test] + fn count_min_f64_and_mode_in_metadata_round_trip() { + // f64 counters (fractional weights) are the other wire-eligible config. + let mut sketch = CountMin::, FastPath>::with_dimensions(4, 16); + sketch.insert_many(&DataInput::U64(1), 2.5); + sketch.insert_many(&DataInput::U64(2), 1.25); + + let encoded = sketch.serialize_to_bytes().expect("serialize"); + let decoded = CountMin::, FastPath>::deserialize_from_bytes(&encoded) + .expect("deserialize"); + assert_eq!( + sketch.as_storage().as_slice(), + decoded.as_storage().as_slice() + ); + + // Counter type + mode are pinned by the target: an f64/fast payload must + // not decode into an i64/regular sketch (metadata mismatch). + assert!(CountMin::, RegularPath>::deserialize_from_bytes(&encoded).is_err()); + } + + /// Fail closed (not panic) on a crafted envelope with a zero dimension: + /// valid envelope + valid metadata that carries `cols == 0`, with an empty + /// `[counts]` payload. Before the guard this panicked in `Vector2D::from_fn` + /// via `0.ilog2()`. + #[test] + fn count_min_rejects_zero_dimension_payload() { + let metadata = + rmp_serde::to_vec_named(&cms_metadata::(4, 0, "i64", "regular")) + .unwrap(); + let payload = rmp_serde::to_vec(&CmsPayload:: { counts: Vec::new() }).unwrap(); + let bytes = envelope::encode(CMS_KIND, &metadata, &payload); + assert!( + CountMin::, RegularPath>::deserialize_from_bytes(&bytes).is_err(), + "zero-dimension metadata must be rejected, not panic" + ); + } + + /// Fail closed on an unexpected metadata key (mirrors the HLL test). + #[test] + fn cms_metadata_rejects_unknown_keys() { + #[derive(Serialize)] + struct WithExtra { + metadata_version: u8, + hash_profile_id: String, + hash_algorithm: String, + seed_derivation: String, + input_encoding: String, + seed_list: Vec, + matrix_seed_index: u32, + rows: u32, + cols: u32, + counter_type: String, + mode: String, + bogus_field: u8, // key not in CmsMetadata + } + let m = cms_metadata::(2, 3, "i64", "regular"); + let extra = WithExtra { + metadata_version: m.metadata_version, + hash_profile_id: m.hash_profile_id.clone(), + hash_algorithm: m.hash_algorithm.clone(), + seed_derivation: m.seed_derivation.clone(), + input_encoding: m.input_encoding.clone(), + seed_list: m.seed_list.clone(), + matrix_seed_index: m.matrix_seed_index, + rows: m.rows, + cols: m.cols, + counter_type: m.counter_type.clone(), + mode: m.mode.clone(), + bogus_field: 7, + }; + let bytes = rmp_serde::to_vec_named(&extra).unwrap(); + assert!(rmp_serde::from_slice::(&bytes).is_err()); + } +} diff --git a/src/sketches/countsketch.rs b/src/sketches/countsketch.rs index 196b422..22a56cc 100644 --- a/src/sketches/countsketch.rs +++ b/src/sketches/countsketch.rs @@ -467,7 +467,7 @@ impl Count, M, H> { /// Human-friendly helper used by the serializer demo binaries. pub fn debug(&self) { for row in 0..self.counts.rows() { - println!("row {}: {:?}", row, self.counts.row_slice(row)); + println!("row {row}: {:?}", self.counts.row_slice(row)); } } } diff --git a/src/sketches/ddsketch.rs b/src/sketches/ddsketch.rs index 4c8fdd3..3d96208 100644 --- a/src/sketches/ddsketch.rs +++ b/src/sketches/ddsketch.rs @@ -587,13 +587,7 @@ mod tests { let err = rel_err(got, want); assert!( err <= tol, - "quantile {} (p={:.2}) relerr={:.4} got={} want={} tol={}", - name, - p, - err, - got, - want, - tol + "quantile {name} (p={p:.2}) relerr={err:.4} got={got} want={want} tol={tol}" ); } } @@ -654,13 +648,7 @@ mod tests { let err = rel_err(got, want); assert!( err <= tol, - "quantile {} (p={:.2}) relerr={:.4} got={} want={} tol={}", - name, - p, - err, - got, - want, - tol + "quantile {name} (p={p:.2}) relerr={err:.4} got={got} want={want} tol={tol}" ); } } @@ -724,13 +712,7 @@ mod tests { let err = rel_err(got, want); assert!( err <= tol, - "quantile {} (p={:.2}) relerr={:.4} got={} want={} tol={}", - name, - p, - err, - got, - want, - tol + "quantile {name} (p={p:.2}) relerr={err:.4} got={got} want={want} tol={tol}" ); } } @@ -791,13 +773,7 @@ mod tests { let err = rel_err(got, want); assert!( err <= tol + 1e-9, - "quantile {} (p={:.2}) relerr={:.4} got={} want={} tol={}", - name, - p, - err, - got, - want, - tol + "quantile {name} (p={p:.2}) relerr={err:.4} got={got} want={want} tol={tol}" ); } } diff --git a/src/sketches/hll.rs b/src/sketches/hll.rs index dcdbae3..3a7397c 100644 --- a/src/sketches/hll.rs +++ b/src/sketches/hll.rs @@ -37,12 +37,16 @@ use crate::structures::fixed_structure::{ HllBucketListP12, HllBucketListP14, HllBucketListP16, HllRegisterStorage, }; use crate::{CANONICAL_HASH_SEED, DataInput, DefaultXxHasher, SketchHasher, hash64_seeded}; -use rmp_serde::{ - decode::Error as RmpDecodeError, encode::Error as RmpEncodeError, from_slice, to_vec_named, -}; use serde::{Deserialize, Serialize}; use std::marker::PhantomData; +mod wire; +pub use wire::HllWireVariant; +pub(crate) use wire::{ + HLL_KIND_CLASSIC, HLL_KIND_ERTL_MLE, HLL_KIND_HIP, HllMetadata, HllPayloadHip, HllPayloadPlain, + standard_hll_metadata, +}; + /// Generic HyperLogLog sketch parameterized by estimation variant, register storage, and hasher. #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(bound = "")] @@ -116,16 +120,6 @@ impl } } - /// Serializes the sketch into MessagePack bytes. - pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - to_vec_named(self) - } - - /// Deserializes a sketch from MessagePack bytes. - pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - from_slice(bytes) - } - /// Borrow the raw register byte slice (one byte per register). pub fn registers_as_slice(&self) -> &[u8] { self.registers.as_slice() @@ -368,16 +362,6 @@ impl HyperLogLogHIPImpl { pub fn estimate(&self) -> usize { self.est as usize } - - /// Serializes the sketch into MessagePack bytes. - pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - to_vec_named(self) - } - - /// Deserializes a sketch from MessagePack bytes. - pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { - from_slice(bytes) - } } // DataInput adapters for HIP (hashing + batch helpers). @@ -450,7 +434,6 @@ mod tests { const TARGETS: [usize; 7] = [10, 100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000]; const ERROR_TOLERANCE: f64 = 0.02; const P12_ERROR_TOLERANCE: f64 = 0.03; - const SERDE_SAMPLE: usize = 100_000; #[test] fn hll_child_insert_emits_on_improvement() { @@ -476,13 +459,6 @@ mod tests { fn merge_into(&mut self, other: &Self); } - trait HllSerializable: HllEstimator { - fn serialize_to_bytes(&self) -> Result, RmpEncodeError>; - fn deserialize_from_bytes(bytes: &[u8]) -> Result - where - Self: Sized; - } - impl HllEstimator for HyperLogLogImpl { @@ -511,18 +487,6 @@ mod tests { } } - impl HllSerializable - for HyperLogLogImpl - { - fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - HyperLogLogImpl::::serialize_to_bytes(self) - } - - fn deserialize_from_bytes(bytes: &[u8]) -> Result { - HyperLogLogImpl::::deserialize_from_bytes(bytes) - } - } - macro_rules! impl_ertl_mle_test_traits { ($storage:ty) => { impl HllEstimator for HyperLogLogImpl { @@ -548,16 +512,6 @@ mod tests { self.merge(other); } } - - impl HllSerializable for HyperLogLogImpl { - fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - HyperLogLogImpl::::serialize_to_bytes(self) - } - - fn deserialize_from_bytes(bytes: &[u8]) -> Result { - HyperLogLogImpl::::deserialize_from_bytes(bytes) - } - } }; } @@ -582,16 +536,6 @@ mod tests { } } - impl HllSerializable for HyperLogLogHIPImpl { - fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { - HyperLogLogHIPImpl::::serialize_to_bytes(self) - } - - fn deserialize_from_bytes(bytes: &[u8]) -> Result { - HyperLogLogHIPImpl::::deserialize_from_bytes(bytes) - } - } - #[test] fn hyperloglog_accuracy_within_two_percent() { assert_accuracy::>("HyperLogLog"); @@ -645,36 +589,6 @@ mod tests { assert_merge_accuracy_within::>("HllErtlP12", P12_ERROR_TOLERANCE); } - #[test] - fn hyperloglog_round_trip_serialization() { - assert_serialization_round_trip::>("HyperLogLog"); - } - - #[test] - fn hll_ertl_round_trip_serialization() { - assert_serialization_round_trip::>("HllErtl"); - } - - #[test] - fn hllds_round_trip_serialization() { - assert_serialization_round_trip::("HllDs"); - } - - #[test] - fn hyperloglog_p12_round_trip_serialization() { - assert_serialization_round_trip::>("HyperLogLogP12"); - } - - #[test] - fn hll_ertl_p12_round_trip_serialization() { - assert_serialization_round_trip::>("HllErtlP12"); - } - - #[test] - fn hllds_p12_round_trip_serialization() { - assert_serialization_round_trip::("HllDsP12"); - } - // insert 10 values and check corresponding counter is updated #[test] fn hll_correctness_test() { @@ -849,42 +763,4 @@ mod tests { ); } } - - fn assert_serialization_round_trip(name: &str) - where - S: HllSerializable, - { - let mut sketch = S::default(); - for value in 0..SERDE_SAMPLE { - let input = DataInput::U64(value as u64); - sketch.push(&input); - } - - let encoded = sketch - .serialize_to_bytes() - .unwrap_or_else(|err| panic!("{name} serialize_to_bytes failed: {err}")); - assert!( - !encoded.is_empty(), - "{name} serialization output should not be empty" - ); - - let decoded = S::deserialize_from_bytes(&encoded) - .unwrap_or_else(|err| panic!("{name} deserialize_from_bytes failed: {err}")); - - let reencoded = decoded - .serialize_to_bytes() - .unwrap_or_else(|err| panic!("{name} re-serialize failed: {err}")); - - assert_eq!( - encoded, reencoded, - "{name} serialized bytes differed after round trip" - ); - - let original_est = sketch.estimate(); - let decoded_est = decoded.estimate(); - assert!( - (original_est - decoded_est).abs() <= ERROR_TOLERANCE * original_est.max(1.0), - "{name} estimate mismatch after round trip: before {original_est}, after {decoded_est}" - ); - } } diff --git a/src/sketches/hll/wire.rs b/src/sketches/hll/wire.rs new file mode 100644 index 0000000..e3f14b6 --- /dev/null +++ b/src/sketches/hll/wire.rs @@ -0,0 +1,611 @@ +//! ASAPv1 wire serialization for the HyperLogLog sketches. +//! +//! Child submodule of [`crate::sketches::hll`]: it holds ALL of HLL's +//! serialization (the metadata/payload DTOs, kind_id constants, the +//! [`HllWireVariant`] mapping, and the `serialize_to_bytes` / +//! `deserialize_from_bytes` impls) while the algorithm lives in the parent +//! module file (`hll.rs`). Being a descendant module, it can read the sketch structs' private +//! fields (`self.registers`) and construct them directly without widening any +//! field visibility. See `docs/asapv1_wire_format.md`. + +use rmp_serde::{decode::Error as RmpDecodeError, encode::Error as RmpEncodeError, from_slice}; +use serde::{Deserialize, Serialize}; +use std::marker::PhantomData; + +use crate::message_pack_format::envelope; +use crate::structures::fixed_structure::HllRegisterStorage; +use crate::{DefaultXxHasher, HashProfile, SketchHasher}; + +use super::{Classic, ErtlMLE, HyperLogLogHIPImpl, HyperLogLogImpl}; + +/// Maps an in-memory HLL estimator variant to its ASAPv1 `kind_id`. +pub trait HllWireVariant { + /// The `[family, variant]` kind_id for this estimator. + const WIRE_KIND_ID: &'static [u8]; +} + +impl HllWireVariant for Classic { + const WIRE_KIND_ID: &'static [u8] = HLL_KIND_CLASSIC; +} + +impl HllWireVariant for ErtlMLE { + const WIRE_KIND_ID: &'static [u8] = HLL_KIND_ERTL_MLE; +} + +/// HLL payload (ASAPv1 §3.1), serialized as a msgpack **array** (`to_vec`, +/// positional). `registers` is a msgpack `bin` (one byte per register, matching +/// Go's `[]byte`) via `serde_bytes` rather than serde's default `array`. +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct HllPayloadPlain { + #[serde(with = "serde_bytes")] + pub(crate) registers: Vec, +} + +/// HIP payload: the register bin plus the three HIP running scalars. +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct HllPayloadHip { + #[serde(with = "serde_bytes")] + pub(crate) registers: Vec, + pub(crate) hip_kxq0: f64, + pub(crate) hip_kxq1: f64, + pub(crate) hip_est: f64, +} + +const HLL_KIND_FAMILY: u8 = 0x01; +pub(crate) const HLL_KIND_CLASSIC: &[u8] = &[HLL_KIND_FAMILY, 0x01]; +pub(crate) const HLL_KIND_ERTL_MLE: &[u8] = &[HLL_KIND_FAMILY, 0x02]; +pub(crate) const HLL_KIND_HIP: &[u8] = &[HLL_KIND_FAMILY, 0x03]; + +/// Descriptor metadata for an HLL sketch (ASAPv1 §2), serialized as a msgpack +/// **map** (`to_vec_named`) with keys in this declaration order — the canonical +/// order the wire spec fixes. HLL carries the full hash spec (including the +/// inlined `seed_list`, so the bytes are self-describing) plus the seed index it +/// uses and its one structural param, `precision`. `deny_unknown_fields` makes +/// decode fail closed on any unexpected key rather than silently dropping it. +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct HllMetadata { + pub(crate) metadata_version: u8, + pub(crate) hash_profile_id: String, + pub(crate) hash_algorithm: String, + pub(crate) seed_derivation: String, + pub(crate) input_encoding: String, + pub(crate) seed_list: Vec, + pub(crate) canonical_seed_index: u32, + pub(crate) precision: u32, +} + +/// Builds the HLL descriptor metadata from the hasher's [`HashProfile`], so the +/// wire bytes truthfully describe how the sketch was hashed (rather than +/// hardcoding the standard profile). +pub(crate) fn hll_metadata(precision: u32) -> HllMetadata { + HllMetadata { + metadata_version: 1, + hash_profile_id: H::PROFILE_ID.to_string(), + hash_algorithm: H::ALGORITHM.to_string(), + seed_derivation: H::SEED_DERIVATION.to_string(), + input_encoding: H::INPUT_ENCODING.to_string(), + seed_list: H::seed_list(), + canonical_seed_index: H::CANONICAL_SEED_INDEX, + precision, + } +} + +/// The standard ProjectASAP profile metadata (the [`DefaultXxHasher`] profile). +/// Used by the portable path, which only represents standard-profile sketches. +pub(crate) fn standard_hll_metadata(precision: u32) -> HllMetadata { + hll_metadata::(precision) +} + +/// Validate the envelope for a known target and return the raw payload bytes. +/// The metadata is checked against the profile of `H`, so bytes hashed under a +/// different profile are rejected (fail closed). +fn validated_hll_payload<'a, H: HashProfile>( + bytes: &'a [u8], + expected_kind_id: &[u8], + expected_precision: u32, +) -> Result<&'a [u8], RmpDecodeError> { + let (kind_id, metadata, payload) = + envelope::split(bytes).map_err(RmpDecodeError::Uncategorized)?; + if kind_id != expected_kind_id { + return Err(RmpDecodeError::Uncategorized(format!( + "HLL kind_id mismatch: stored {kind_id:?}, expected {expected_kind_id:?}" + ))); + } + let meta: HllMetadata = from_slice(metadata)?; + if meta != hll_metadata::(expected_precision) { + return Err(RmpDecodeError::Uncategorized( + "ASAPv1 HLL envelope: metadata mismatch".to_string(), + )); + } + Ok(payload) +} + +/// Rebuild register storage from the payload's register bin. +fn registers_from_bytes( + registers: &[u8], +) -> Result { + if registers.len() != Registers::NUM_REGISTERS { + return Err(RmpDecodeError::Uncategorized(format!( + "HLL register length mismatch: stored {}, expected {}", + registers.len(), + Registers::NUM_REGISTERS + ))); + } + let mut out = Registers::default(); + out.as_mut_slice().copy_from_slice(registers); + Ok(out) +} + +// Wire serialization for the generic HyperLogLog estimator family. `wire` is a +// descendant of the sketch module, so these impls read the private `registers` +// field and construct the struct directly. +impl + HyperLogLogImpl +{ + /// Serializes the sketch into an ASAPv1 MessagePack envelope. The metadata is + /// derived from the hasher's [`HashProfile`], so it truthfully describes how + /// the sketch was hashed. + pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> + where + Variant: HllWireVariant, + H: HashProfile, + { + let metadata = rmp_serde::to_vec_named(&hll_metadata::(Registers::PRECISION as u32))?; + let payload = rmp_serde::to_vec(&HllPayloadPlain { + registers: self.registers.as_slice().to_vec(), + })?; + Ok(envelope::encode(Variant::WIRE_KIND_ID, &metadata, &payload)) + } + + /// Deserializes a sketch from an ASAPv1 MessagePack envelope. Bytes whose + /// metadata does not match this hasher's [`HashProfile`] are rejected. + pub fn deserialize_from_bytes(bytes: &[u8]) -> Result + where + Variant: HllWireVariant, + H: HashProfile, + { + let payload = + validated_hll_payload::(bytes, Variant::WIRE_KIND_ID, Registers::PRECISION as u32)?; + let payload: HllPayloadPlain = from_slice(payload)?; + let registers = registers_from_bytes::(&payload.registers)?; + Ok(Self { + registers, + _marker: PhantomData, + _hasher: PhantomData, + }) + } +} + +// Wire serialization for the HIP estimator. +impl HyperLogLogHIPImpl { + /// Serializes the sketch into an ASAPv1 MessagePack envelope. + pub fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { + let metadata = + rmp_serde::to_vec_named(&standard_hll_metadata(Registers::PRECISION as u32))?; + let payload = rmp_serde::to_vec(&HllPayloadHip { + registers: self.registers.as_slice().to_vec(), + hip_kxq0: self.kxq0, + hip_kxq1: self.kxq1, + hip_est: self.est, + })?; + Ok(envelope::encode(HLL_KIND_HIP, &metadata, &payload)) + } + + /// Deserializes a sketch from an ASAPv1 MessagePack envelope. + pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { + let payload = validated_hll_payload::( + bytes, + HLL_KIND_HIP, + Registers::PRECISION as u32, + )?; + let payload: HllPayloadHip = from_slice(payload)?; + let registers = registers_from_bytes::(&payload.registers)?; + Ok(Self { + registers, + kxq0: payload.hip_kxq0, + kxq1: payload.hip_kxq1, + est: payload.hip_est, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::sketches::hll::{HyperLogLog, HyperLogLogHIP, HyperLogLogHIPP12, HyperLogLogP12}; + use crate::structures::fixed_structure::{HllBucketListP12, HllBucketListP14}; + use crate::{DataInput, HllBucketList}; + + const ERROR_TOLERANCE: f64 = 0.02; + const SERDE_SAMPLE: usize = 100_000; + + // Minimal test harness (mirrors the algorithm-side one in `mod.rs`) so the + // serialization round-trip tests can drive any (variant × precision) sketch + // generically. + trait HllEstimator: Default { + fn push(&mut self, input: &DataInput); + fn estimate(&self) -> f64; + } + + trait HllSerializable: HllEstimator { + fn serialize_to_bytes(&self) -> Result, RmpEncodeError>; + fn deserialize_from_bytes(bytes: &[u8]) -> Result + where + Self: Sized; + } + + impl HllEstimator + for HyperLogLogImpl + { + fn push(&mut self, input: &DataInput) { + self.insert(input); + } + + fn estimate(&self) -> f64 { + HyperLogLogImpl::::estimate(self) as f64 + } + } + + impl HllSerializable + for HyperLogLogImpl + { + fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { + HyperLogLogImpl::::serialize_to_bytes(self) + } + + fn deserialize_from_bytes(bytes: &[u8]) -> Result { + HyperLogLogImpl::::deserialize_from_bytes(bytes) + } + } + + macro_rules! impl_ertl_mle_wire_test_traits { + ($storage:ty) => { + impl HllEstimator for HyperLogLogImpl { + fn push(&mut self, input: &DataInput) { + self.insert(input); + } + + fn estimate(&self) -> f64 { + HyperLogLogImpl::::estimate(self) as f64 + } + } + + impl HllSerializable + for HyperLogLogImpl + { + fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { + HyperLogLogImpl::::serialize_to_bytes(self) + } + + fn deserialize_from_bytes(bytes: &[u8]) -> Result { + HyperLogLogImpl::::deserialize_from_bytes(bytes) + } + } + }; + } + + impl_ertl_mle_wire_test_traits!(HllBucketListP12); + impl_ertl_mle_wire_test_traits!(HllBucketListP14); + + impl HllEstimator for HyperLogLogHIPImpl { + fn push(&mut self, input: &DataInput) { + self.insert(input); + } + + fn estimate(&self) -> f64 { + HyperLogLogHIPImpl::::estimate(self) as f64 + } + } + + impl HllSerializable for HyperLogLogHIPImpl { + fn serialize_to_bytes(&self) -> Result, RmpEncodeError> { + HyperLogLogHIPImpl::::serialize_to_bytes(self) + } + + fn deserialize_from_bytes(bytes: &[u8]) -> Result { + HyperLogLogHIPImpl::::deserialize_from_bytes(bytes) + } + } + + fn assert_serialization_round_trip(name: &str) + where + S: HllSerializable, + { + let mut sketch = S::default(); + for value in 0..SERDE_SAMPLE { + let input = DataInput::U64(value as u64); + sketch.push(&input); + } + + let encoded = sketch + .serialize_to_bytes() + .unwrap_or_else(|err| panic!("{name} serialize_to_bytes failed: {err}")); + assert!( + !encoded.is_empty(), + "{name} serialization output should not be empty" + ); + + let decoded = S::deserialize_from_bytes(&encoded) + .unwrap_or_else(|err| panic!("{name} deserialize_from_bytes failed: {err}")); + + let reencoded = decoded + .serialize_to_bytes() + .unwrap_or_else(|err| panic!("{name} re-serialize failed: {err}")); + + assert_eq!( + encoded, reencoded, + "{name} serialized bytes differed after round trip" + ); + + let original_est = sketch.estimate(); + let decoded_est = decoded.estimate(); + assert!( + (original_est - decoded_est).abs() <= ERROR_TOLERANCE * original_est.max(1.0), + "{name} estimate mismatch after round trip: before {original_est}, after {decoded_est}" + ); + } + + #[test] + fn hyperloglog_round_trip_serialization() { + assert_serialization_round_trip::>("HyperLogLog"); + } + + #[test] + fn hll_ertl_round_trip_serialization() { + assert_serialization_round_trip::>("HllErtl"); + } + + #[test] + fn hllds_round_trip_serialization() { + assert_serialization_round_trip::("HllDs"); + } + + #[test] + fn hyperloglog_p12_round_trip_serialization() { + assert_serialization_round_trip::>("HyperLogLogP12"); + } + + #[test] + fn hll_ertl_p12_round_trip_serialization() { + assert_serialization_round_trip::>("HllErtlP12"); + } + + #[test] + fn hllds_p12_round_trip_serialization() { + assert_serialization_round_trip::("HllDsP12"); + } + + #[test] + fn hll_envelope_structure_and_kind_id_guard() { + // ASAPv1 envelope header for an Ertl-MLE sketch: magic, version 0x01, + // kind_id_len 2, kind_id [0x01, 0x02]. + let mut sketch = HyperLogLog::::default(); + for value in 0..1000 { + sketch.insert(&DataInput::U64(value)); + } + let bytes = sketch.serialize_to_bytes().expect("serialize"); + + assert!(bytes.starts_with(envelope::MAGIC)); + assert_eq!(bytes[6], envelope::VERSION); + assert_eq!(bytes[7], 2, "kind_id_len"); + assert_eq!(&bytes[8..10], HLL_KIND_ERTL_MLE); + + let decoded = HyperLogLog::::deserialize_from_bytes(&bytes).expect("decode"); + assert_eq!(decoded.registers_as_slice(), sketch.registers_as_slice()); + + // A Classic decoder must reject Ertl-MLE bytes (kind_id mismatch). + assert!(HyperLogLog::::deserialize_from_bytes(&bytes).is_err()); + } + + #[test] + fn hll_hip_round_trip_preserves_state() { + let mut sketch = HyperLogLogHIP::default(); + for value in 0..1000 { + sketch.insert(&DataInput::U64(value)); + } + let bytes = sketch.serialize_to_bytes().expect("serialize"); + assert!(bytes.starts_with(envelope::MAGIC)); + assert_eq!(&bytes[8..10], HLL_KIND_HIP); + + let decoded = HyperLogLogHIP::deserialize_from_bytes(&bytes).expect("decode"); + assert_eq!(decoded.registers.as_slice(), sketch.registers.as_slice()); + assert_eq!(decoded.kxq0, sketch.kxq0); + assert_eq!(decoded.kxq1, sketch.kxq1); + assert_eq!(decoded.est, sketch.est); + } + + /// Drift guard: the native path and the (deprecated) portable path must emit + /// byte-identical ASAPv1 envelopes. Keep until golden byte-vectors exist. + #[test] + fn native_and_portable_hll_bytes_match() { + use crate::message_pack_format::MessagePackCodec; + use crate::message_pack_format::portable::hll::{HllSketch, HllVariant}; + + // Ertl-MLE / Datafusion: registers-only payload. + let mut native = HyperLogLog::::default(); + for v in 0..1000 { + native.insert(&DataInput::U64(v)); + } + let native_bytes = native.serialize_to_bytes().expect("native serialize"); + let portable = HllSketch::from_raw( + HllVariant::Datafusion, + HllBucketList::PRECISION as u32, + native.registers_as_slice().to_vec(), + 0.0, + 0.0, + 0.0, + ); + assert_eq!( + native_bytes, + portable.to_msgpack().expect("portable serialize") + ); + + // HIP: registers + three running scalars. + let mut hip = HyperLogLogHIP::default(); + for v in 0..1000 { + hip.insert(&DataInput::U64(v)); + } + let hip_bytes = hip.serialize_to_bytes().expect("native serialize"); + let portable_hip = HllSketch::from_raw( + HllVariant::Hip, + HllBucketList::PRECISION as u32, + hip.registers.as_slice().to_vec(), + hip.kxq0, + hip.kxq1, + hip.est, + ); + assert_eq!( + hip_bytes, + portable_hip.to_msgpack().expect("portable serialize") + ); + } + + // A test-only custom hasher: it hashes exactly like `DefaultXxHasher` + // (delegation) but declares a DIFFERENT `HashProfile` (distinct profile id + // and seed list). Serialization is derived from the profile, so an + // `AltHasher` sketch serializes truthfully and its metadata differs from the + // standard profile on the wire. + // + // Note: that an *unprofiled* hasher cannot be serialized is a compile-time + // guarantee — `serialize_to_bytes`/`deserialize_from_bytes` are bounded on + // `H: HashProfile`, so a hasher that impls only `SketchHasher` fails to + // compile. There is nothing to assert at runtime. + #[derive(Clone, Debug)] + struct AltHasher; + + impl SketchHasher for AltHasher { + type HashType = ::HashType; + + fn hash64_seeded(d: usize, key: &DataInput) -> u64 { + DefaultXxHasher::hash64_seeded(d, key) + } + fn hash128_seeded(d: usize, key: &DataInput) -> u128 { + DefaultXxHasher::hash128_seeded(d, key) + } + fn hash_item64_seeded(d: usize, key: &crate::HeapItem) -> u64 { + DefaultXxHasher::hash_item64_seeded(d, key) + } + fn hash_item128_seeded(d: usize, key: &crate::HeapItem) -> u128 { + DefaultXxHasher::hash_item128_seeded(d, key) + } + fn hash_for_matrix_seeded( + seed_idx: usize, + rows: usize, + cols: usize, + key: &DataInput, + ) -> Self::HashType { + DefaultXxHasher::hash_for_matrix_seeded(seed_idx, rows, cols, key) + } + } + + impl HashProfile for AltHasher { + const PROFILE_ID: &'static str = "test.alt.profile.v1"; + const ALGORITHM: &'static str = "xxh3_64_128"; + const SEED_DERIVATION: &'static str = "seed_list_index_wrap"; + const INPUT_ENCODING: &'static str = "projectasap.input.v1"; + fn seed_list() -> Vec { + // A deliberately different seed list so the wire bytes differ. + vec![1, 2, 3, 4, 5] + } + const CANONICAL_SEED_INDEX: u32 = crate::CANONICAL_HASH_SEED as u32; + const MATRIX_SEED_INDEX: u32 = 0; + } + + #[test] + fn hll_custom_hasher_profile_round_trips_and_is_self_describing() { + // (a) An HLL built with a custom-profile hasher round-trips. + let mut alt = HyperLogLogImpl::::default(); + let mut std = HyperLogLog::::default(); + for v in 0..1000 { + alt.insert(&DataInput::U64(v)); + std.insert(&DataInput::U64(v)); + } + let alt_bytes = alt.serialize_to_bytes().expect("alt serialize"); + let decoded = + HyperLogLogImpl::::deserialize_from_bytes( + &alt_bytes, + ) + .expect("alt decode"); + assert_eq!(decoded.registers_as_slice(), alt.registers_as_slice()); + + // (b) Its bytes differ from the DefaultXxHasher sketch's bytes: the + // metadata is derived from the (different) profile, so it is not a lie. + let std_bytes = std.serialize_to_bytes().expect("std serialize"); + assert_ne!( + alt_bytes, std_bytes, + "a custom profile must serialize different metadata than the standard profile" + ); + + // (c) Decoding AltHasher bytes into a DefaultXxHasher-typed HLL fails + // closed (profile mismatch), never silently accepted. + assert!( + HyperLogLog::::deserialize_from_bytes(&alt_bytes).is_err(), + "standard-profile decode must reject custom-profile bytes" + ); + } + + /// Fail closed: metadata carrying an unexpected key must be rejected, not + /// silently dropped (`deny_unknown_fields`). + #[test] + fn hll_metadata_rejects_unknown_keys() { + #[derive(Serialize)] + struct WithExtra { + metadata_version: u8, + hash_profile_id: String, + hash_algorithm: String, + seed_derivation: String, + input_encoding: String, + seed_list: Vec, + canonical_seed_index: u32, + precision: u32, + bogus_field: u8, // key not in HllMetadata + } + let std = standard_hll_metadata(14); + let extra = WithExtra { + metadata_version: std.metadata_version, + hash_profile_id: std.hash_profile_id.clone(), + hash_algorithm: std.hash_algorithm.clone(), + seed_derivation: std.seed_derivation.clone(), + input_encoding: std.input_encoding.clone(), + seed_list: std.seed_list.clone(), + canonical_seed_index: std.canonical_seed_index, + precision: std.precision, + bogus_field: 7, + }; + let bytes = rmp_serde::to_vec_named(&extra).expect("encode"); + assert!( + rmp_serde::from_slice::(&bytes).is_err(), + "an unexpected metadata key must be rejected" + ); + } + + /// A decoder rejects bytes whose precision differs from the target storage: + /// P12 bytes must not decode into a P14-typed sketch (metadata mismatch). + #[test] + fn hll_precision_cross_rejection() { + let mut p12 = HyperLogLogP12::::default(); + for v in 0..100 { + p12.insert(&DataInput::U64(v)); + } + let bytes = p12.serialize_to_bytes().expect("serialize"); + assert!( + HyperLogLog::::deserialize_from_bytes(&bytes).is_err(), + "P12 bytes must be rejected by a P14 decoder" + ); + } + + /// A Classic decoder rejects HIP bytes (kind_id mismatch). + #[test] + fn hll_hip_kind_id_rejected_by_classic() { + let mut hip = HyperLogLogHIP::default(); + for v in 0..100 { + hip.insert(&DataInput::U64(v)); + } + let bytes = hip.serialize_to_bytes().expect("serialize"); + assert!( + HyperLogLog::::deserialize_from_bytes(&bytes).is_err(), + "HIP bytes must be rejected by a Classic decoder" + ); + } +} diff --git a/src/sketches/kll.rs b/src/sketches/kll.rs index f8f9cde..cef5f51 100644 --- a/src/sketches/kll.rs +++ b/src/sketches/kll.rs @@ -1296,7 +1296,7 @@ mod tests { let median = cdf.query(0.5); // Median should be 30.5 - assert!(median > 20.0 && median < 40.2, "Median = {}", median); + assert!(median > 20.0 && median < 40.2, "Median = {median}"); // Test error handling for non-numeric input let result = kll.update_data_input(&DataInput::String("not a number".to_string())); @@ -1326,7 +1326,7 @@ mod tests { // cdf.print_entries(); let median = cdf.query(0.5); // only 30 and 40 is possible - assert!(median == 30.0 || median == 40.0, "Median = {}", median); + assert!(median == 30.0 || median == 40.0, "Median = {median}"); } #[test] @@ -1351,7 +1351,7 @@ mod tests { // kll.print_compactors(); let median = cdf.query(0.5); // Median should be 30 - assert!(median == 30.0, "Median = {}", median); + assert!(median == 30.0, "Median = {median}"); } #[test] diff --git a/src/sketches/kll_dynamic.rs b/src/sketches/kll_dynamic.rs index 25463f2..6459a95 100644 --- a/src/sketches/kll_dynamic.rs +++ b/src/sketches/kll_dynamic.rs @@ -732,7 +732,7 @@ mod tests { let median = cdf.query(0.5); // Median should be 30.5 - assert!(median > 20.0 && median < 40.2, "Median = {}", median); + assert!(median > 20.0 && median < 40.2, "Median = {median}"); // Test error handling for non-numeric input let result = kll.update_data_input(&DataInput::String("not a number".to_string())); @@ -755,7 +755,7 @@ mod tests { let cdf = kll.cdf(); let median = cdf.query(0.5); // only 30 and 40 is possible - assert!(median == 30.0 || median == 40.0, "Median = {}", median); + assert!(median == 30.0 || median == 40.0, "Median = {median}"); } #[test] @@ -772,7 +772,7 @@ mod tests { let cdf = kll.cdf(); let median = cdf.query(0.5); // Median should be 30 - assert!(median == 30.0, "Median = {}", median); + assert!(median == 30.0, "Median = {median}"); } #[test] diff --git a/tests/asapv1_golden.rs b/tests/asapv1_golden.rs new file mode 100644 index 0000000..a3fe105 --- /dev/null +++ b/tests/asapv1_golden.rs @@ -0,0 +1,153 @@ +//! Cross-language ASAPv1 golden byte-vector tests. +//! +//! Each fixture is built from a fixed, KNOWN raw sketch state (register bytes / +//! matrix values set directly, never hashed) and asserted to serialize to the +//! exact bytes checked into `asapv1_golden/*.hex`. The same `.hex` files live +//! byte-identically in `sketchlib-go/asapv1_golden/`, where the Go test suite +//! proves its encoder emits the same bytes. Together they machine-prove the Rust +//! and Go ASAPv1 wire encodings are byte-identical for these configs. +//! +//! See `asapv1_golden/README.md`. + +use asap_sketchlib::{ + Classic, CountMin, ErtlMLE, FastPath, HllSketch, HllVariant, HyperLogLogHIPP12, HyperLogLogP12, + MessagePackCodec, RegularPath, Vector2D, +}; + +fn decode_hex(s: &str) -> Vec { + let s = s.trim(); + assert!(s.len() % 2 == 0, "golden hex must have even length"); + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16).expect("valid hex")) + .collect() +} + +const GOLDEN_CLASSIC: &str = include_str!("../asapv1_golden/hll_classic_p12.hex"); +const GOLDEN_ERTL: &str = include_str!("../asapv1_golden/hll_ertl_mle_p12.hex"); +const GOLDEN_HIP: &str = include_str!("../asapv1_golden/hll_hip_p12.hex"); +const GOLDEN_CMS_I64: &str = include_str!("../asapv1_golden/cms_i64_regular_2x3.hex"); +const GOLDEN_CMS_F64: &str = include_str!("../asapv1_golden/cms_f64_fast_2x3.hex"); + +/// The known P12 register pattern shared by all three HLL fixtures. +fn p12_registers() -> Vec { + let mut r = vec![0u8; 4096]; + r[0] = 1; + r[1] = 7; + r[100] = 42; + r[4095] = 3; + r +} + +const I64_VALS: [[i64; 3]; 2] = [[0, 1, 127], [128, 300, 65536]]; +const F64_VALS: [[f64; 3]; 2] = [[0.0, 1.5, 2.25], [3.75, 4.125, 5.0625]]; + +// --------------------------------------------------------------------------- +// HLL: build known state -> serialize == golden, and golden round-trips. +// --------------------------------------------------------------------------- + +#[test] +fn hll_classic_p12_matches_golden() { + let want = decode_hex(GOLDEN_CLASSIC); + let regs = p12_registers(); + + // Build known state and serialize. + let got = HllSketch::from_raw(HllVariant::Regular, 12, regs.clone(), 0.0, 0.0, 0.0) + .to_msgpack() + .expect("serialize"); + assert_eq!(got, want, "Classic P12 bytes diverge from golden"); + + // Native wire path (src/sketches/hll.rs) round-trips the golden identically. + let native = HyperLogLogP12::::deserialize_from_bytes(&want).expect("native decode"); + assert_eq!(native.registers_as_slice(), regs.as_slice()); + assert_eq!(native.serialize_to_bytes().expect("re-serialize"), want); + + // Portable decode round-trips to the same known state. + let decoded = HllSketch::from_msgpack(&want).expect("decode"); + assert_eq!(decoded.registers, regs); + assert_eq!(decoded.precision, 12); + assert_eq!(decoded.variant, HllVariant::Regular); +} + +#[test] +fn hll_ertl_mle_p12_matches_golden() { + let want = decode_hex(GOLDEN_ERTL); + let regs = p12_registers(); + + let got = HllSketch::from_raw(HllVariant::Datafusion, 12, regs.clone(), 0.0, 0.0, 0.0) + .to_msgpack() + .expect("serialize"); + assert_eq!(got, want, "Ertl-MLE P12 bytes diverge from golden"); + + let native = HyperLogLogP12::::deserialize_from_bytes(&want).expect("native decode"); + assert_eq!(native.registers_as_slice(), regs.as_slice()); + assert_eq!(native.serialize_to_bytes().expect("re-serialize"), want); + + let decoded = HllSketch::from_msgpack(&want).expect("decode"); + assert_eq!(decoded.registers, regs); + assert_eq!(decoded.variant, HllVariant::Datafusion); +} + +#[test] +fn hll_hip_p12_matches_golden() { + let want = decode_hex(GOLDEN_HIP); + let regs = p12_registers(); + + let got = HllSketch::from_raw(HllVariant::Hip, 12, regs.clone(), 1.5, 2.5, 3.0) + .to_msgpack() + .expect("serialize"); + assert_eq!(got, want, "HIP P12 bytes diverge from golden"); + + // Native HIP wire path round-trips the golden identically. + let native = HyperLogLogHIPP12::deserialize_from_bytes(&want).expect("native decode"); + assert_eq!(native.serialize_to_bytes().expect("re-serialize"), want); + + let decoded = HllSketch::from_msgpack(&want).expect("decode"); + assert_eq!(decoded.registers, regs); + assert_eq!(decoded.variant, HllVariant::Hip); + assert_eq!(decoded.hip_kxq0, 1.5); + assert_eq!(decoded.hip_kxq1, 2.5); + assert_eq!(decoded.hip_est, 3.0); +} + +// --------------------------------------------------------------------------- +// Count-Min: build known matrix state -> serialize == golden, round-trips. +// --------------------------------------------------------------------------- + +#[test] +fn cms_i64_regular_2x3_matches_golden() { + let want = decode_hex(GOLDEN_CMS_I64); + + let sketch = + CountMin::, RegularPath>::from_storage(Vector2D::from_fn(2, 3, |r, c| { + I64_VALS[r][c] + })); + let got = sketch.serialize_to_bytes().expect("serialize"); + assert_eq!(got, want, "CMS i64/regular bytes diverge from golden"); + + let decoded = + CountMin::, RegularPath>::deserialize_from_bytes(&want).expect("decode"); + let flat: Vec = I64_VALS.iter().flatten().copied().collect(); + assert_eq!(decoded.as_storage().as_slice(), flat.as_slice()); + assert_eq!(decoded.rows(), 2); + assert_eq!(decoded.cols(), 3); + assert_eq!(decoded.serialize_to_bytes().expect("re-serialize"), want); +} + +#[test] +fn cms_f64_fast_2x3_matches_golden() { + let want = decode_hex(GOLDEN_CMS_F64); + + let sketch = + CountMin::, FastPath>::from_storage(Vector2D::from_fn(2, 3, |r, c| { + F64_VALS[r][c] + })); + let got = sketch.serialize_to_bytes().expect("serialize"); + assert_eq!(got, want, "CMS f64/fast bytes diverge from golden"); + + let decoded = + CountMin::, FastPath>::deserialize_from_bytes(&want).expect("decode"); + let flat: Vec = F64_VALS.iter().flatten().copied().collect(); + assert_eq!(decoded.as_storage().as_slice(), flat.as_slice()); + assert_eq!(decoded.serialize_to_bytes().expect("re-serialize"), want); +} diff --git a/tests/xtest_consumer.rs b/tests/xtest_consumer.rs index ce9364e..ea9332a 100644 --- a/tests/xtest_consumer.rs +++ b/tests/xtest_consumer.rs @@ -62,7 +62,7 @@ fn cross_language_proto() { let cm_state = match env.sketch_state { Some(sketch_envelope::SketchState::CountMin(ref s)) => s.clone(), - other => panic!("expected CountMin sketch_state, got {:?}", other), + other => panic!("expected CountMin sketch_state, got {other:?}"), }; println!( @@ -96,10 +96,7 @@ fn cross_language_proto() { let hot_key = b"item:42" as &[u8]; let hot_hash = xxh3_64_seeded(SEED_0, hot_key); - println!( - "[CountMin] Step 3/3 — Point query 'item:42' (hash=0x{:016x})", - hot_hash - ); + println!("[CountMin] Step 3/3 — Point query 'item:42' (hash=0x{hot_hash:016x})"); let bits_per_row = col_bits(cols); let mask = (cols as u64) - 1; @@ -112,12 +109,9 @@ fn cross_language_proto() { min_freq = v; } } - println!( - "[CountMin] min frequency estimate = {:.0} (expect ≥ 101)", - min_freq - ); + println!("[CountMin] min frequency estimate = {min_freq:.0} (expect ≥ 101)"); if min_freq < 101.0 { - eprintln!("[CountMin] FAIL: frequency {:.0} < 101", min_freq); + eprintln!("[CountMin] FAIL: frequency {min_freq:.0} < 101"); all_ok = false; } else { println!("[CountMin] PASS"); @@ -139,7 +133,7 @@ fn cross_language_proto() { let kll_state = match env.sketch_state { Some(sketch_envelope::SketchState::Kll(ref s)) => s.clone(), - other => panic!("expected KLL sketch_state, got {:?}", other), + other => panic!("expected KLL sketch_state, got {other:?}"), }; println!( @@ -154,17 +148,14 @@ fn cross_language_proto() { let kll = KllFromProto::from_state(&kll_state); let p50 = kll.quantile(0.50); let p99 = kll.quantile(0.99); - println!( - "[KLL] Step 3/3 — p50 ≈ {:.1} (expect ~5000) p99 ≈ {:.1} (expect ~9900)", - p50, p99 - ); + println!("[KLL] Step 3/3 — p50 ≈ {p50:.1} (expect ~5000) p99 ≈ {p99:.1} (expect ~9900)"); let p50_ok = (p50 - 5000.0).abs() / 5000.0 < 0.05; let p99_ok = (p99 - 9900.0).abs() / 9900.0 < 0.05; if p50_ok && p99_ok { println!("[KLL] PASS"); } else { - eprintln!("[KLL] FAIL: p50={:.1} p99={:.1}", p50, p99); + eprintln!("[KLL] FAIL: p50={p50:.1} p99={p99:.1}"); all_ok = false; } @@ -184,7 +175,7 @@ fn cross_language_proto() { let dd_state = match env.sketch_state { Some(sketch_envelope::SketchState::Ddsketch(ref s)) => s.clone(), - other => panic!("expected DDSketch sketch_state, got {:?}", other), + other => panic!("expected DDSketch sketch_state, got {other:?}"), }; // `count` was dropped from the wire (ProjectASAP/sketchlib-go#243); @@ -216,7 +207,7 @@ fn cross_language_proto() { if p50_ok && p99_ok { println!("[DDSketch] PASS"); } else { - eprintln!("[DDSketch] FAIL: p50={:?} p99={:?}", p50_dd, p99_dd); + eprintln!("[DDSketch] FAIL: p50={p50_dd:?} p99={p99_dd:?}"); all_ok = false; } @@ -236,7 +227,7 @@ fn cross_language_proto() { let hll_state = match env.sketch_state { Some(sketch_envelope::SketchState::Hll(ref s)) => s.clone(), - other => panic!("expected HLL sketch_state, got {:?}", other), + other => panic!("expected HLL sketch_state, got {other:?}"), }; println!( @@ -247,14 +238,11 @@ fn cross_language_proto() { ); let hll_card = hll_ertl_mle_estimate(&hll_state); - println!( - "[HLL] Step 3/3 — cardinality ≈ {} (expect ~50000)", - hll_card - ); + println!("[HLL] Step 3/3 — cardinality ≈ {hll_card} (expect ~50000)"); if (40_000..=65_000).contains(&hll_card) { println!("[HLL] PASS"); } else { - eprintln!("[HLL] FAIL: cardinality {} not in [40000, 65000]", hll_card); + eprintln!("[HLL] FAIL: cardinality {hll_card} not in [40000, 65000]"); all_ok = false; } @@ -274,7 +262,7 @@ fn cross_language_proto() { let cs_state = match env.sketch_state { Some(sketch_envelope::SketchState::CountSketch(ref s)) => s.clone(), - other => panic!("expected CountSketch sketch_state, got {:?}", other), + other => panic!("expected CountSketch sketch_state, got {other:?}"), }; println!( @@ -288,14 +276,11 @@ fn cross_language_proto() { // Go hash: common.Hash64([]byte("cs:hot")) = xxh3_64_seeded(seedList[0], b"cs:hot") let cs_hot_hash = xxh3_64_seeded(SEED_0, b"cs:hot"); let cs_est = count_sketch_query_float(&cs_state, cs_hot_hash); - println!( - "[CountSketch] Step 3/3 — 'cs:hot' est = {:.0} (expect ≥ 200)", - cs_est - ); + println!("[CountSketch] Step 3/3 — 'cs:hot' est = {cs_est:.0} (expect ≥ 200)"); if cs_est >= 200.0 { println!("[CountSketch] PASS"); } else { - eprintln!("[CountSketch] FAIL: estimate {:.0} < 200", cs_est); + eprintln!("[CountSketch] FAIL: estimate {cs_est:.0} < 200"); all_ok = false; } @@ -315,7 +300,7 @@ fn cross_language_proto() { let coco_state = match env.sketch_state { Some(sketch_envelope::SketchState::Coco(ref s)) => s.clone(), - other => panic!("expected CocoSketch sketch_state, got {:?}", other), + other => panic!("expected CocoSketch sketch_state, got {other:?}"), }; println!( @@ -330,14 +315,11 @@ fn cross_language_proto() { // DeriveIndex(hash, row, width): col = (hash >> (row * maskBitsForWidth(width))) & (width-1) let coco_hash = xxh3_64_seeded(SEED_0, b"coco:hot"); let coco_est = coco_estimate(&coco_state, coco_hash); - println!( - "[CocoSketch] Step 3/3 — 'coco:hot' est = {} (expect ≥ 500)", - coco_est - ); + println!("[CocoSketch] Step 3/3 — 'coco:hot' est = {coco_est} (expect ≥ 500)"); if coco_est >= 500 { println!("[CocoSketch] PASS"); } else { - eprintln!("[CocoSketch] FAIL: estimate {} < 500", coco_est); + eprintln!("[CocoSketch] FAIL: estimate {coco_est} < 500"); all_ok = false; } @@ -357,7 +339,7 @@ fn cross_language_proto() { let elastic_state = match env.sketch_state { Some(sketch_envelope::SketchState::Elastic(ref s)) => s.clone(), - other => panic!("expected ElasticSketch sketch_state, got {:?}", other), + other => panic!("expected ElasticSketch sketch_state, got {other:?}"), }; println!( @@ -371,14 +353,11 @@ fn cross_language_proto() { // Go uses CanonicalHashSeed = seedList[5] = 0x6a09e667 let elephant_hash = xxh3_64_seeded(SEED_5, b"elephant"); let elephant_est = elastic_query(&elastic_state, "elephant", elephant_hash); - println!( - "[ElasticSketch] Step 3/3 — 'elephant' est = {} (expect ≥ 900)", - elephant_est - ); + println!("[ElasticSketch] Step 3/3 — 'elephant' est = {elephant_est} (expect ≥ 900)"); if elephant_est >= 900 { println!("[ElasticSketch] PASS"); } else { - eprintln!("[ElasticSketch] FAIL: estimate {} < 900", elephant_est); + eprintln!("[ElasticSketch] FAIL: estimate {elephant_est} < 900"); all_ok = false; } @@ -398,7 +377,7 @@ fn cross_language_proto() { let um_state = match env.sketch_state { Some(sketch_envelope::SketchState::Univmon(ref s)) => s.clone(), - other => panic!("expected UnivMon sketch_state, got {:?}", other), + other => panic!("expected UnivMon sketch_state, got {other:?}"), }; println!( @@ -409,17 +388,11 @@ fn cross_language_proto() { let um_card = univmon_cardinality(&um_state); // Note: the g-sum heuristic typically underestimates (Go itself reports ~4250 for 10k inserts). // We verify the Rust result matches Go's algorithm rather than the true cardinality. - println!( - "[UnivMon] Step 3/3 — cardinality ≈ {:.0} (g-sum heuristic, Go also ~4250)", - um_card - ); + println!("[UnivMon] Step 3/3 — cardinality ≈ {um_card:.0} (g-sum heuristic, Go also ~4250)"); if (1_000.0..=15_000.0).contains(&um_card) { println!("[UnivMon] PASS"); } else { - eprintln!( - "[UnivMon] FAIL: cardinality {:.0} not in [1000, 15000]", - um_card - ); + eprintln!("[UnivMon] FAIL: cardinality {um_card:.0} not in [1000, 15000]"); all_ok = false; } @@ -439,7 +412,7 @@ fn cross_language_proto() { let hydra_state = match env.sketch_state { Some(sketch_envelope::SketchState::Hydra(ref s)) => s.clone(), - other => panic!("expected Hydra sketch_state, got {:?}", other), + other => panic!("expected Hydra sketch_state, got {other:?}"), }; println!( @@ -456,14 +429,11 @@ fn cross_language_proto() { let hydra_subkey_hash = xxh3_64_seeded(SEED_6, b"hydra:42"); let hydra_value_hash = xxh3_64_seeded(SEED_0, b"hydra:42"); let hydra_est = hydra_query_cm(&hydra_state, hydra_subkey_hash, hydra_value_hash); - println!( - "[Hydra] Step 3/3 — 'hydra:42' est = {:.0} (expect ≥ 51)", - hydra_est - ); + println!("[Hydra] Step 3/3 — 'hydra:42' est = {hydra_est:.0} (expect ≥ 51)"); if hydra_est >= 51.0 { println!("[Hydra] PASS"); } else { - eprintln!("[Hydra] FAIL: estimate {:.0} < 51", hydra_est); + eprintln!("[Hydra] FAIL: estimate {hydra_est:.0} < 51"); all_ok = false; } @@ -488,7 +458,7 @@ fn cross_language_proto() { let cm_state = match env.sketch_state { Some(sketch_envelope::SketchState::CountMin(ref s)) => s.clone(), - other => panic!("expected CountMin sketch_state, got {:?}", other), + other => panic!("expected CountMin sketch_state, got {other:?}"), }; let rows = cm_state.rows as usize; let cols = cm_state.cols as usize; @@ -541,7 +511,7 @@ fn cross_language_proto() { let mut hll_state = match env.sketch_state { Some(sketch_envelope::SketchState::Hll(ref s)) => s.clone(), - other => panic!("expected HLL sketch_state, got {:?}", other), + other => panic!("expected HLL sketch_state, got {other:?}"), }; // Dual-read the registers (sampling thins them → may be sparse-encoded). let regs =