From adeecae5ade7848d30ae379f68fe8a0c8d236176 Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Mon, 6 Jul 2026 12:07:38 +0100 Subject: [PATCH] Add bit iteration benchmarks Signed-off-by: Robert Kruszewski --- Cargo.lock | 1 + encodings/runend/Cargo.toml | 4 + encodings/runend/benches/run_end_filter.rs | 113 +++++++++++++++++ encodings/runend/src/compute/filter.rs | 2 +- encodings/runend/src/lib.rs | 1 + vortex-array/Cargo.toml | 4 + vortex-array/benches/validity_is_valid.rs | 71 +++++++++++ vortex-duckdb/Cargo.toml | 5 + vortex-duckdb/benches/bool_export.rs | 58 +++++++++ vortex-mask/Cargo.toml | 8 ++ vortex-mask/benches/mask_iteration.rs | 141 +++++++++++++++++++++ vortex-mask/benches/valid_counts.rs | 51 ++++++++ 12 files changed, 458 insertions(+), 1 deletion(-) create mode 100644 encodings/runend/benches/run_end_filter.rs create mode 100644 vortex-array/benches/validity_is_valid.rs create mode 100644 vortex-duckdb/benches/bool_export.rs create mode 100644 vortex-mask/benches/mask_iteration.rs create mode 100644 vortex-mask/benches/valid_counts.rs diff --git a/Cargo.lock b/Cargo.lock index e232ebdeb66..af0d17a2c00 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10217,6 +10217,7 @@ dependencies = [ "bitvec", "cbindgen", "cc", + "codspeed-divan-compat", "custom-labels", "futures", "geo-types", diff --git a/encodings/runend/Cargo.toml b/encodings/runend/Cargo.toml index 268aa6ac3ca..c7b38d2494f 100644 --- a/encodings/runend/Cargo.toml +++ b/encodings/runend/Cargo.toml @@ -56,3 +56,7 @@ harness = false [[bench]] name = "run_end_take" harness = false + +[[bench]] +name = "run_end_filter" +harness = false diff --git a/encodings/runend/benches/run_end_filter.rs b/encodings/runend/benches/run_end_filter.rs new file mode 100644 index 00000000000..395b41063b1 --- /dev/null +++ b/encodings/runend/benches/run_end_filter.rs @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Benchmarks for the run-end filter inner loop (`filter_run_end_primitive`). +//! +//! This measures the kernel directly rather than going through the lazy +//! `ArrayRef::filter` (which only builds a `FilterArray` node and does not run +//! the kernel). The hot work is a per-run popcount of the predicate mask, which +//! now uses `BitBuffer::count_range` (SIMD) instead of a bit-by-bit walk. + +#![expect(clippy::cast_possible_truncation)] +#![expect(clippy::cast_precision_loss)] +#![expect(clippy::cast_sign_loss)] +#![expect(clippy::expect_used)] + +use std::fmt; + +use divan::Bencher; +use rand::SeedableRng; +use rand::rngs::StdRng; +use rand::seq::SliceRandom; +use vortex_buffer::BitBuffer; +use vortex_runend::_benchmarking::filter_run_end_primitive; + +fn main() { + divan::main(); +} + +#[derive(Clone, Copy)] +struct FilterBenchArgs { + /// Total logical length of the decoded array. + length: usize, + /// Average run length used when building the run-end array. + run_length: usize, + /// Fraction of mask bits that are set to `true`. + density: f64, +} + +impl fmt::Display for FilterBenchArgs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "len={}_run={}_density={:.1}", + self.length, self.run_length, self.density + ) + } +} + +const FILTER_ARGS: &[FilterBenchArgs] = &[ + FilterBenchArgs { + length: 4_096, + run_length: 16, + density: 0.1, + }, + FilterBenchArgs { + length: 4_096, + run_length: 16, + density: 0.5, + }, + FilterBenchArgs { + length: 4_096, + run_length: 16, + density: 0.9, + }, + FilterBenchArgs { + length: 16_384, + run_length: 16, + density: 0.1, + }, + FilterBenchArgs { + length: 16_384, + run_length: 16, + density: 0.5, + }, + FilterBenchArgs { + length: 16_384, + run_length: 16, + density: 0.9, + }, +]; + +/// Build the run-end boundaries (cumulative run lengths) for `length` rows. +fn build_run_ends(length: usize, run_length: usize) -> Vec { + let n_runs = length.div_ceil(run_length); + (0..n_runs) + .map(|r| (((r + 1) * run_length).min(length)) as u32) + .collect() +} + +/// Build a predicate mask of `length` bits with approximately `density` set bits, +/// shuffled so the set bits are spread across runs. +fn build_mask(length: usize, density: f64) -> BitBuffer { + let n_true = (length as f64 * density).round() as usize; + let mut bits = vec![false; length]; + for b in bits.iter_mut().take(n_true) { + *b = true; + } + let mut rng = StdRng::seed_from_u64(0x5eed); + bits.shuffle(&mut rng); + BitBuffer::from(bits) +} + +#[divan::bench(args = FILTER_ARGS)] +fn filter_run_end(bencher: Bencher, args: FilterBenchArgs) { + let run_ends = build_run_ends(args.length, args.run_length); + let mask = build_mask(args.length, args.density); + let length = args.length as u64; + bencher + .with_inputs(|| (run_ends.clone(), mask.clone())) + .bench_refs(|(run_ends, mask)| { + filter_run_end_primitive::(run_ends, 0, length, mask).expect("filter") + }); +} diff --git a/encodings/runend/src/compute/filter.rs b/encodings/runend/src/compute/filter.rs index ca99fdf1912..c460dd6552d 100644 --- a/encodings/runend/src/compute/filter.rs +++ b/encodings/runend/src/compute/filter.rs @@ -74,7 +74,7 @@ impl FilterKernel for RunEnd { } // Code adapted from apache arrow-rs https://github.com/apache/arrow-rs/blob/b1f5c250ebb6c1252b4e7c51d15b8e77f4c361fa/arrow-select/src/filter.rs#L425 -fn filter_run_end_primitive + AsPrimitive>( +pub fn filter_run_end_primitive + AsPrimitive>( run_ends: &[R], offset: u64, length: u64, diff --git a/encodings/runend/src/lib.rs b/encodings/runend/src/lib.rs index 8169f5d8dfc..a0b26b54dab 100644 --- a/encodings/runend/src/lib.rs +++ b/encodings/runend/src/lib.rs @@ -21,6 +21,7 @@ mod rules; #[doc(hidden)] pub mod _benchmarking { + pub use compute::filter::filter_run_end_primitive; pub use compute::take::take_indices_unchecked; use super::*; diff --git a/vortex-array/Cargo.toml b/vortex-array/Cargo.toml index c83397b50d8..4d8ef4ea2bb 100644 --- a/vortex-array/Cargo.toml +++ b/vortex-array/Cargo.toml @@ -200,6 +200,10 @@ required-features = ["_test-harness"] name = "dict_mask" harness = false +[[bench]] +name = "validity_is_valid" +harness = false + [[bench]] name = "dict_unreferenced_mask" harness = false diff --git a/vortex-array/benches/validity_is_valid.rs b/vortex-array/benches/validity_is_valid.rs new file mode 100644 index 00000000000..dc22fcb5478 --- /dev/null +++ b/vortex-array/benches/validity_is_valid.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Benchmarks the cost of checking array-backed [`Validity`] per element. +//! +//! For the `Validity::Array` variant, `Validity::execute_is_valid` runs a scalar lookup through +//! the compute stack on *every* call, and the (now deprecated) `is_valid` additionally spins up a +//! fresh `ExecutionCtx` per call. Either way, calling it in a `for i in 0..n` loop is +//! pathologically slow. The fix used by callers is to materialize the validity into a `Mask` once +//! (`execute_mask`) and then do cheap O(1) bit reads via `Mask::value`. This bench contrasts the two. +//! +//! Sizes are kept small because the per-element variant is intentionally the slow one we are +//! measuring. + +#![allow(clippy::unwrap_used)] + +use std::sync::LazyLock; + +use divan::Bencher; +use vortex_array::VortexSessionExecute; +use vortex_array::array_session; +use vortex_array::validity::Validity; +use vortex_buffer::BitBuffer; +use vortex_session::VortexSession; + +fn main() { + divan::main(); +} + +const SIZES: &[usize] = &[256, 1024]; + +/// Build an array-backed validity (~10% nulls so it is `Validity::Array`). +fn array_validity(len: usize) -> Validity { + Validity::from(BitBuffer::from_iter( + (0..len).map(|i| !i.is_multiple_of(10)), + )) +} + +static SESSION: LazyLock = LazyLock::new(array_session); + +/// Per-element validity check over array-backed validity (the antipattern). This mirrors the +/// deprecated `Validity::is_valid(i)`: a fresh `ExecutionCtx` plus a scalar lookup on every call. +#[divan::bench(args = SIZES)] +fn is_valid_per_element(bencher: Bencher, len: usize) { + let validity = array_validity(len); + bencher + .with_inputs(|| (&validity, SESSION.create_execution_ctx())) + .bench_refs(|(validity, ctx)| { + let mut count = 0usize; + for i in 0..len { + count += validity.execute_is_valid(i, ctx).unwrap() as usize; + } + count + }); +} + +/// Materialize the validity into a `Mask` once, then read bits (the fix). +#[divan::bench(args = SIZES)] +fn execute_mask_then_value(bencher: Bencher, len: usize) { + let validity = array_validity(len); + bencher + .with_inputs(|| (&validity, SESSION.create_execution_ctx())) + .bench_refs(|(validity, ctx)| { + let mask = validity.execute_mask(len, ctx).unwrap(); + let mut count = 0usize; + for i in 0..len { + count += mask.value(i) as usize; + } + count + }); +} diff --git a/vortex-duckdb/Cargo.toml b/vortex-duckdb/Cargo.toml index b6896910a89..01a0e114fde 100644 --- a/vortex-duckdb/Cargo.toml +++ b/vortex-duckdb/Cargo.toml @@ -45,6 +45,7 @@ vortex-utils = { workspace = true, features = ["dashmap"] } [dev-dependencies] anyhow = { workspace = true } +divan = { workspace = true } geo-types = { workspace = true } jiff = { workspace = true } rstest = { workspace = true } @@ -54,6 +55,10 @@ vortex-runend = { workspace = true } vortex-sequence = { workspace = true } wkb = { workspace = true } +[[bench]] +name = "bool_export" +harness = false + [lints] workspace = true diff --git a/vortex-duckdb/benches/bool_export.rs b/vortex-duckdb/benches/bool_export.rs new file mode 100644 index 00000000000..f8094b554ae --- /dev/null +++ b/vortex-duckdb/benches/bool_export.rs @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Microbenchmark for the bit -> byte-bool unpack performed by `BoolExporter::export`. +//! +//! DuckDB stores booleans as one byte per value (`&mut [bool]`), while Vortex stores them +//! bit-packed in a [`BitBuffer`]. On every boolean column export we unpack a slice of the +//! bit buffer into the destination byte slice. +//! +//! This bench isolates that pure unpacking logic so it can run without a live DuckDB +//! vector: a reused `Vec` stands in for the DuckDB byte-bool destination. It compares +//! the previous implementation (`.iter().collect::>()` followed by +//! `copy_from_slice`) against the new allocation-free direct zip-write. + +use divan::Bencher; +use vortex::buffer::BitBuffer; + +fn main() { + divan::main(); +} + +const SIZES: &[usize] = &[1024, 16_384]; + +/// Bit densities (true ratio), expressed as `(numerator, denominator)`. +const DENSITIES: &[(usize, usize)] = &[(1, 2), (1, 10), (9, 10)]; + +fn make_buffer(len: usize, density: (usize, usize)) -> BitBuffer { + let (num, den) = density; + // `+ 8` so we can slice off a non-byte-aligned offset, matching real exports. + BitBuffer::from_iter((0..len + 8).map(|i| (i % den) < num)) +} + +/// Previous implementation: allocate a throwaway `Vec` then copy into the destination. +#[divan::bench(args = SIZES, consts = [0usize, 1usize, 2usize])] +fn old_collect_copy(bencher: Bencher, len: usize) { + let buffer = make_buffer(len, DENSITIES[DENSITY_IDX]); + bencher + .with_inputs(|| vec![false; len]) + .bench_refs(|dst: &mut Vec| { + // Offset by 1 to exercise a non-byte-aligned slice like the export hot path. + dst.copy_from_slice(&buffer.slice(1..(1 + len)).iter().collect::>()); + divan::black_box(&dst); + }); +} + +/// New implementation: zip the sliced bit iterator directly into the destination slice. +#[divan::bench(args = SIZES, consts = [0usize, 1usize, 2usize])] +fn new_zip_write(bencher: Bencher, len: usize) { + let buffer = make_buffer(len, DENSITIES[DENSITY_IDX]); + bencher + .with_inputs(|| vec![false; len]) + .bench_refs(|dst: &mut Vec| { + for (slot, bit) in dst.iter_mut().zip(buffer.slice(1..(1 + len)).iter()) { + *slot = bit; + } + divan::black_box(&dst); + }); +} diff --git a/vortex-mask/Cargo.toml b/vortex-mask/Cargo.toml index 7aa5bb2ef85..cf681101d43 100644 --- a/vortex-mask/Cargo.toml +++ b/vortex-mask/Cargo.toml @@ -37,5 +37,13 @@ harness = false name = "rank" harness = false +[[bench]] +name = "valid_counts" +harness = false + +[[bench]] +name = "mask_iteration" +harness = false + [lints] workspace = true diff --git a/vortex-mask/benches/mask_iteration.rs b/vortex-mask/benches/mask_iteration.rs new file mode 100644 index 00000000000..50f5c321303 --- /dev/null +++ b/vortex-mask/benches/mask_iteration.rs @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Strategy comparison for "process the valid elements of a mask". +//! +//! The kernel is `sum of values[i] for every set bit i` — a stand-in for any +//! validity-gated loop. We compare: +//! +//! * `per_element_value` — `for i in 0..n { if buf.value(i) {..} }` (what callers +//! typically write after materializing validity into a mask) +//! * `bit_iterator` — arrow's `BitIterator` (tracks a word internally) +//! * `word_trailing_zeros`— iterate `u64` words, fast-path all-set/all-unset, and +//! walk set bits with `trailing_zeros` / `w &= w - 1` +//! * `set_slices` — iterate contiguous true runs (`set_slices`) +//! * `set_indices` — iterate set positions (`set_indices`) +//! +//! Run across densities to expose the dense-vs-sparse crossover. + +#![allow(clippy::unwrap_used, clippy::cast_possible_truncation)] + +use divan::Bencher; +use vortex_buffer::BitBuffer; + +fn main() { + divan::main(); +} + +const ARGS: &[(usize, f64)] = &[ + (16_384, 0.01), + (16_384, 0.10), + (16_384, 0.50), + (16_384, 0.90), + (16_384, 0.99), +]; + +fn make(len: usize, density: f64) -> (BitBuffer, Vec) { + let threshold = (density * 1000.0) as usize; + let buf = BitBuffer::from_iter((0..len).map(|i| (i * 7 + 13) % 1000 < threshold)); + let values = (0..len as u64).collect(); + (buf, values) +} + +#[divan::bench(args = ARGS)] +fn per_element_value(bencher: Bencher, (len, density): (usize, f64)) { + let (buf, values) = make(len, density); + bencher + .with_inputs(|| (&buf, &values)) + .bench_refs(|(buf, values)| { + let mut acc = 0u64; + for i in 0..len { + if buf.value(i) { + acc = acc.wrapping_add(values[i]); + } + } + acc + }); +} + +#[divan::bench(args = ARGS)] +fn bit_iterator(bencher: Bencher, (len, density): (usize, f64)) { + let (buf, values) = make(len, density); + bencher + .with_inputs(|| (&buf, &values)) + .bench_refs(|(buf, values)| { + let mut acc = 0u64; + for (i, set) in buf.iter().enumerate() { + if set { + acc = acc.wrapping_add(values[i]); + } + } + acc + }); +} + +#[divan::bench(args = ARGS)] +fn word_trailing_zeros(bencher: Bencher, (len, density): (usize, f64)) { + let (buf, values) = make(len, density); + bencher + .with_inputs(|| (&buf, &values)) + .bench_refs(|(buf, values)| { + let mut acc = 0u64; + let chunks = buf.chunks(); + let mut base = 0usize; + for word in chunks.iter() { + if word == u64::MAX { + for k in 0..64 { + acc = acc.wrapping_add(values[base + k]); + } + } else { + let mut w = word; + while w != 0 { + let b = w.trailing_zeros() as usize; + acc = acc.wrapping_add(values[base + b]); + w &= w - 1; + } + } + base += 64; + } + let mut w = chunks.remainder_bits(); + let rem = buf.len() - base; + while w != 0 { + let b = w.trailing_zeros() as usize; + if b >= rem { + break; + } + acc = acc.wrapping_add(values[base + b]); + w &= w - 1; + } + acc + }); +} + +#[divan::bench(args = ARGS)] +fn set_slices(bencher: Bencher, (len, density): (usize, f64)) { + let (buf, values) = make(len, density); + bencher + .with_inputs(|| (&buf, &values)) + .bench_refs(|(buf, values)| { + let mut acc = 0u64; + for (start, end) in buf.set_slices() { + for i in start..end { + acc = acc.wrapping_add(values[i]); + } + } + acc + }); +} + +#[divan::bench(args = ARGS)] +fn set_indices(bencher: Bencher, (len, density): (usize, f64)) { + let (buf, values) = make(len, density); + bencher + .with_inputs(|| (&buf, &values)) + .bench_refs(|(buf, values)| { + let mut acc = 0u64; + for i in buf.set_indices() { + acc = acc.wrapping_add(values[i]); + } + acc + }); +} diff --git a/vortex-mask/benches/valid_counts.rs b/vortex-mask/benches/valid_counts.rs new file mode 100644 index 00000000000..425a021fa6f --- /dev/null +++ b/vortex-mask/benches/valid_counts.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Benchmarks for `Mask::valid_counts_for_indices`. +//! +//! This mirrors the hot path in the pco/zstd slice decoders, which call +//! `valid_counts_for_indices(&[slice_start, slice_stop])` to translate a row +//! range into a value range. The cost is dominated by counting set bits in the +//! prefix `[0, slice_stop)`, so a SIMD popcount over the bit buffer should beat +//! a bit-by-bit walk handily for large `slice_stop`. + +#![allow(clippy::unwrap_used, clippy::cast_possible_truncation)] + +use divan::Bencher; +use vortex_buffer::BitBuffer; +use vortex_mask::Mask; + +fn main() { + divan::main(); +} + +const BENCH_SIZES: &[(usize, f64)] = &[(4_096, 0.1), (4_096, 0.9), (16_384, 0.1), (16_384, 0.9)]; + +fn create_mask(len: usize, density: f64) -> Mask { + Mask::from_buffer(BitBuffer::from_iter((0..len).map(|i| { + let threshold = (density * 1000.0) as usize; + (i * 7 + 13) % 1000 < threshold + }))) +} + +/// The pco/zstd slice pattern: two indices bracketing a sub-range. The prefix +/// up to `slice_stop` must be counted, so `slice_stop` near the end is worst case. +#[divan::bench(args = BENCH_SIZES)] +fn slice_bounds(bencher: Bencher, (len, density): (usize, f64)) { + let mask = create_mask(len, density); + let indices = [len / 4, len - len / 8]; + bencher + .with_inputs(|| (&mask, indices)) + .bench_refs(|(mask, indices)| mask.valid_counts_for_indices(indices)); +} + +/// Many monotonically increasing indices spread across the whole mask. +#[divan::bench(args = BENCH_SIZES)] +fn many_indices(bencher: Bencher, (len, density): (usize, f64)) { + let mask = create_mask(len, density); + let stride = (len / 256).max(1); + let indices: Vec = (0..len).step_by(stride).collect(); + bencher + .with_inputs(|| (&mask, indices.as_slice())) + .bench_refs(|(mask, indices)| mask.valid_counts_for_indices(indices)); +}