Skip to content

Commit aefe065

Browse files
committed
Address review feedback for percentile_cont(DISTINCT) accumulator
Follow-up to the DistinctPercentileContAccumulator fix, addressing post-merge review comments: - Use the fast foldhash RandomState for the `counts` map instead of the standard library's default SipHash, matching GenericDistinctBuffer. - Compute size() via estimate_memory_size, mirroring the other distinct accumulators. - Add a null-free fast path to update_batch and retract_batch that skips the per-element validity check when the input array has no nulls. - Add ORDER BY id to the sliding-window regression query in aggregate.slt so its output row order is deterministic. Co-authored-by: Claude Code
1 parent 994fc81 commit aefe065

2 files changed

Lines changed: 34 additions & 8 deletions

File tree

datafusion/functions-aggregate/src/percentile_cont.rs

Lines changed: 32 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,10 @@ use arrow::{
3232
use num_traits::AsPrimitive;
3333

3434
use arrow::array::ArrowNativeTypeOp;
35+
use datafusion_common::hash_utils::RandomState;
3536
use datafusion_common::internal_err;
3637
use datafusion_common::types::{NativeType, logical_float64};
38+
use datafusion_common::utils::memory::estimate_memory_size;
3739
use datafusion_functions_aggregate_common::noop_accumulator::NoopAccumulator;
3840

3941
use crate::min_max::{max_udaf, min_udaf};
@@ -672,7 +674,11 @@ where
672674
#[derive(Debug)]
673675
struct DistinctPercentileContAccumulator<T: ArrowNumericType> {
674676
/// Distinct value -> number of in-window rows carrying it.
675-
counts: HashMap<Hashable<T::Native>, usize>,
677+
///
678+
/// Uses the same fast (foldhash) `RandomState` as the shared
679+
/// `GenericDistinctBuffer` rather than the standard library's default
680+
/// SipHash, which is considerably slower for this hot path.
681+
counts: HashMap<Hashable<T::Native>, usize, RandomState>,
676682
percentile: f64,
677683
}
678684

@@ -709,8 +715,15 @@ where
709715
// `values` may carry extra argument columns (e.g. the percentile
710716
// literal); only the first column holds the aggregated values.
711717
let arr = values[0].as_primitive::<T>();
712-
for value in arr.iter().flatten() {
713-
*self.counts.entry(Hashable(value)).or_default() += 1;
718+
if arr.null_count() > 0 {
719+
for value in arr.iter().flatten() {
720+
*self.counts.entry(Hashable(value)).or_default() += 1;
721+
}
722+
} else {
723+
// Fast path: no nulls, so skip the per-element validity check.
724+
for value in arr.values().iter() {
725+
*self.counts.entry(Hashable(*value)).or_default() += 1;
726+
}
714727
}
715728
Ok(())
716729
}
@@ -733,9 +746,11 @@ where
733746
}
734747

735748
fn size(&self) -> usize {
736-
size_of_val(self)
737-
+ self.counts.capacity()
738-
* (size_of::<Hashable<T::Native>>() + size_of::<usize>())
749+
estimate_memory_size::<(Hashable<T::Native>, usize)>(
750+
self.counts.capacity(),
751+
size_of_val(self),
752+
)
753+
.unwrap()
739754
}
740755

741756
fn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
@@ -744,13 +759,23 @@ where
744759
}
745760

746761
let arr = values[0].as_primitive::<T>();
747-
for value in arr.iter().flatten() {
762+
let mut decrement = |value: T::Native| {
748763
if let Some(count) = self.counts.get_mut(&Hashable(value)) {
749764
*count -= 1;
750765
if *count == 0 {
751766
self.counts.remove(&Hashable(value));
752767
}
753768
}
769+
};
770+
if arr.null_count() > 0 {
771+
for value in arr.iter().flatten() {
772+
decrement(value);
773+
}
774+
} else {
775+
// Fast path: no nulls, so skip the per-element validity check.
776+
for value in arr.values().iter() {
777+
decrement(*value);
778+
}
754779
}
755780
Ok(())
756781
}

datafusion/sqllogictest/test_files/aggregate.slt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1122,7 +1122,8 @@ SELECT percentile_cont(DISTINCT x, 0.5) FROM distinct_pct;
11221122
query IR
11231123
SELECT id, percentile_cont(DISTINCT x, 0.5)
11241124
OVER (ORDER BY id ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)
1125-
FROM distinct_pct;
1125+
FROM distinct_pct
1126+
ORDER BY id;
11261127
----
11271128
1 5
11281129
2 5

0 commit comments

Comments
 (0)