Skip to content

perf: outline column desc for min max #10389

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
MassivePizza wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

perf: outline column desc for min max #10389

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d825b8f
outline access to ColumnDescriptor in get_min_max
MassivePizza Jul 16, 2026
d35f98a
inline(always) to help out min_max.
MassivePizza Jul 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 18 additions & 10 deletions parquet/src/column/writer/encoder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ use half::f16;
use crate::basic::{ConvertedType, Encoding, LogicalType, Type};
use crate::bloom_filter::Sbbf;
use crate::column::writer::{
compare_greater, fallback_encoding, has_dictionary_support, is_nan, update_max, update_min,
compare_greater_with, fallback_encoding, has_dictionary_support, is_nan_with, update_max,
update_min,
};
use crate::data_type::DataType;
use crate::data_type::private::ParquetValueType;
Expand Down Expand Up @@ -400,23 +401,26 @@ where
T: ParquetValueType + 'a,
I: Iterator<Item = &'a T>,
{
let logical_type = descr.logical_type_ref();
let converted_type = descr.converted_type();

let first = loop {
let next = iter.next()?;
if !is_nan(descr, next) {
if !is_nan_with(logical_type, next) {
break next;
}
};

let mut min = first;
let mut max = first;
for val in iter {
if is_nan(descr, val) {
if is_nan_with(logical_type, val) {
continue;
}
if compare_greater(descr, min, val) {
if compare_greater_with(logical_type, converted_type, min, val) {
min = val;
}
if compare_greater(descr, val, max) {
if compare_greater_with(logical_type, converted_type, val, max) {
max = val;
}
}
Expand All @@ -429,14 +433,18 @@ where
//
// For max, it has similar logic but will be written as 0.0
// (positive zero)
let min = replace_zero(min, descr, -0.0);
let max = replace_zero(max, descr, 0.0);
let min = replace_zero(min, logical_type, -0.0);
let max = replace_zero(max, logical_type, 0.0);

Some((min, max))
}

#[inline]
fn replace_zero<T: ParquetValueType>(val: &T, descr: &ColumnDescriptor, replace: f32) -> T {
#[inline(always)]
fn replace_zero<T: ParquetValueType>(
val: &T,
logical_type_ref: Option<&LogicalType>,
replace: f32,
) -> T {
match T::PHYSICAL_TYPE {
Type::FLOAT if f32::from_le_bytes(val.as_bytes().try_into().unwrap()) == 0.0 => {
T::try_from_le_slice(&f32::to_le_bytes(replace)).unwrap()
Expand All @@ -445,7 +453,7 @@ fn replace_zero<T: ParquetValueType>(val: &T, descr: &ColumnDescriptor, replace:
T::try_from_le_slice(&f64::to_le_bytes(replace as f64)).unwrap()
}
Type::FIXED_LEN_BYTE_ARRAY
if descr.logical_type_ref() == Some(LogicalType::Float16).as_ref()
if logical_type_ref == Some(LogicalType::Float16).as_ref()
&& f16::from_le_bytes(val.as_bytes().try_into().unwrap()) == f16::NEG_ZERO =>
{
T::try_from_le_slice(&f16::to_le_bytes(f16::from_f32(replace))).unwrap()
Expand Down
31 changes: 23 additions & 8 deletions parquet/src/column/writer/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1649,12 +1649,16 @@ fn update_max<T: ParquetValueType>(descr: &ColumnDescriptor, val: &T, max: &mut
update_stat::<T, _>(descr, val, max, |cur| compare_greater(descr, val, cur))
}

#[inline]
#[allow(clippy::eq_op)]
fn is_nan<T: ParquetValueType>(descr: &ColumnDescriptor, val: &T) -> bool {
is_nan_with(descr.logical_type_ref(), val)
}

#[inline(always)]
#[allow(clippy::eq_op)]
fn is_nan_with<T: ParquetValueType>(logical_type_ref: Option<&LogicalType>, val: &T) -> bool {
match T::PHYSICAL_TYPE {
Type::FLOAT | Type::DOUBLE => val != val,
Type::FIXED_LEN_BYTE_ARRAY if descr.logical_type_ref() == Some(&LogicalType::Float16) => {
Type::FIXED_LEN_BYTE_ARRAY if logical_type_ref == Some(&LogicalType::Float16) => {
let val = val.as_bytes();
let val = f16::from_le_bytes([val[0], val[1]]);
val.is_nan()
Expand Down Expand Up @@ -1685,18 +1689,29 @@ fn update_stat<T: ParquetValueType, F>(
}

/// Evaluate `a > b` according to underlying logical type.
#[inline]
fn compare_greater<T: ParquetValueType>(descr: &ColumnDescriptor, a: &T, b: &T) -> bool {
compare_greater_with(descr.logical_type_ref(), descr.converted_type(), a, b)
}

#[inline(always)]
fn compare_greater_with<T: ParquetValueType>(
logical_type_ref: Option<&LogicalType>,
converted_type: ConvertedType,
a: &T,
b: &T,
) -> bool {
match T::PHYSICAL_TYPE {
Type::INT32 | Type::INT64 => {
if let Some(LogicalType::Integer(IntType {
is_signed: false, ..
})) = descr.logical_type_ref()
})) = logical_type_ref
{
// need to compare unsigned
return compare_greater_unsigned_int(a, b);
}

match descr.converted_type() {
match converted_type {
ConvertedType::UINT_8
| ConvertedType::UINT_16
| ConvertedType::UINT_32
Expand All @@ -1707,13 +1722,13 @@ fn compare_greater<T: ParquetValueType>(descr: &ColumnDescriptor, a: &T, b: &T)
};
}
Type::FIXED_LEN_BYTE_ARRAY | Type::BYTE_ARRAY => {
if let Some(LogicalType::Decimal(_)) = descr.logical_type_ref() {
if let Some(LogicalType::Decimal(_)) = logical_type_ref {
return compare_greater_byte_array_decimals(a.as_bytes(), b.as_bytes());
}
if let ConvertedType::DECIMAL = descr.converted_type() {
if let ConvertedType::DECIMAL = converted_type {
return compare_greater_byte_array_decimals(a.as_bytes(), b.as_bytes());
}
if let Some(LogicalType::Float16) = descr.logical_type_ref() {
if let Some(LogicalType::Float16) = logical_type_ref {
return compare_greater_f16(a.as_bytes(), b.as_bytes());
}
}
Expand Down
Loading