From 70e4069faeea69b5252c6145cf5600e3434a0852 Mon Sep 17 00:00:00 2001 From: albertlockett Date: Sun, 3 May 2026 08:56:02 -0400 Subject: [PATCH 01/33] [arrow-ipc]: dictionary builders for delta - doc fix and integration tests for nested types (#9853) # Which issue does this PR close? - Closes #NNN. # Rationale for this change In https://github.com/apache/arrow-rs/issues/9600 we added capability to call `finish_preserve_values` on any array builder, and it will propagate the choice to preserve dictionary values down into the nested builders. I thought it would be good to extend the integration tests we have in https://github.com/apache/arrow-rs/blob/main/arrow-ipc/tests/test_delta_dictionary.rs to cover the use cases, which was to call the method on builders with nested child builders (such as `StructBuilder` and `ListBuilder`). While reviewing the PR for #9600 I also noticed a small issue with the docs related to using the StreamWriter with delta dictionaries, notably that we set up some options to use delta dictionaries but don't pass the options into the `StreamWriter` constructor: https://docs.rs/arrow-ipc/58.1.0/arrow_ipc/writer/struct.StreamWriter.html#example---efficient-delta-dictionaries # What changes are included in this PR? - adds cases to the integration tests for delta dictionaries covering `ListBuilder` and `StructBuilder` - small docs correction for `StreamWriter` # Are these changes tested? it is simply docs & tests # Are there any user-facing changes? No --- arrow-ipc/src/tests/delta_dictionary.rs | 166 ++++++++++++++++++------ arrow-ipc/src/writer.rs | 2 +- 2 files changed, 129 insertions(+), 39 deletions(-) diff --git a/arrow-ipc/src/tests/delta_dictionary.rs b/arrow-ipc/src/tests/delta_dictionary.rs index dfd8cd33e550..60a6c945a3be 100644 --- a/arrow-ipc/src/tests/delta_dictionary.rs +++ b/arrow-ipc/src/tests/delta_dictionary.rs @@ -24,9 +24,11 @@ use crate::{ writer::FileWriter, }; use arrow_array::{ - Array, ArrayRef, DictionaryArray, RecordBatch, StringArray, builder::StringDictionaryBuilder, + Array, ArrayRef, DictionaryArray, ListArray, RecordBatch, StringArray, StructArray, + builder::{ArrayBuilder, ListBuilder, StringDictionaryBuilder, StructBuilder}, types::Int32Type, }; + use arrow_schema::{DataType, Field, Schema}; use std::io::Cursor; use std::sync::Arc; @@ -35,7 +37,7 @@ use std::sync::Arc; fn test_zero_row_dict() { let batches: &[&[&str]] = &[&[], &["A"], &[], &["B", "C"], &[]]; run_delta_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(vec![]), MessageType::RecordBatch, @@ -48,7 +50,7 @@ fn test_zero_row_dict() { ); run_resend_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(vec![]), MessageType::RecordBatch, @@ -72,7 +74,7 @@ fn test_mixed_delta() { ]; run_delta_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A"])), MessageType::RecordBatch, @@ -87,7 +89,7 @@ fn test_mixed_delta() { ); run_resend_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A"])), MessageType::RecordBatch, @@ -106,7 +108,7 @@ fn test_mixed_delta() { fn test_disjoint_delta() { let batches: &[&[&str]] = &[&["A"], &["B"], &["C", "E"]]; run_delta_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A"])), MessageType::RecordBatch, @@ -118,7 +120,7 @@ fn test_disjoint_delta() { ); run_resend_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A"])), MessageType::RecordBatch, @@ -134,7 +136,7 @@ fn test_disjoint_delta() { fn test_increasing_delta() { let batches: &[&[&str]] = &[&["A"], &["A", "B"], &["A", "B", "C"]]; run_delta_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A"])), MessageType::RecordBatch, @@ -146,7 +148,7 @@ fn test_increasing_delta() { ); run_resend_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A"])), MessageType::RecordBatch, @@ -162,7 +164,7 @@ fn test_increasing_delta() { fn test_single_delta() { let batches: &[&[&str]] = &[&["A", "B", "C"], &["D"]]; run_delta_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A", "B", "C"])), MessageType::RecordBatch, @@ -172,7 +174,7 @@ fn test_single_delta() { ); run_resend_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A", "B", "C"])), MessageType::RecordBatch, @@ -186,7 +188,7 @@ fn test_single_delta() { fn test_single_same_value_sequence() { let batches: &[&[&str]] = &[&["A"], &["A"], &["A"], &["A"]]; run_delta_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A"])), MessageType::RecordBatch, @@ -197,7 +199,7 @@ fn test_single_same_value_sequence() { ); run_resend_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A"])), MessageType::RecordBatch, @@ -216,7 +218,7 @@ fn str_vec(strings: &[&str]) -> Vec { fn test_multi_same_value_sequence() { let batches: &[&[&str]] = &[&["A", "B", "C"], &["A", "B", "C"]]; run_delta_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A", "B", "C"])), MessageType::RecordBatch, @@ -232,17 +234,17 @@ enum MessageType { RecordBatch, } -fn run_resend_sequence_test(batches: &[&[&str]], sequence: &[MessageType]) { +fn run_resend_sequence_test(batches: &[RecordBatch], sequence: &[MessageType]) { let opts = IpcWriteOptions::default().with_dictionary_handling(DictionaryHandling::Resend); run_sequence_test(batches, sequence, opts); } -fn run_delta_sequence_test(batches: &[&[&str]], sequence: &[MessageType]) { +fn run_delta_sequence_test(batches: &[RecordBatch], sequence: &[MessageType]) { let opts = IpcWriteOptions::default().with_dictionary_handling(DictionaryHandling::Delta); run_sequence_test(batches, sequence, opts); } -fn run_sequence_test(batches: &[&[&str]], sequence: &[MessageType], options: IpcWriteOptions) { +fn run_sequence_test(batches: &[RecordBatch], sequence: &[MessageType], options: IpcWriteOptions) { let stream_buf = write_all_to_stream(options.clone(), batches); let ipc_stream = get_ipc_message_stream(stream_buf); for (message, expected) in ipc_stream.iter().zip(sequence.iter()) { @@ -310,7 +312,7 @@ fn test_replace_same_length() { &["A", "B", "C", "D", "E", "F"], &["A", "G", "H", "I", "J", "K"], ]; - run_parity_test(batches); + run_parity_test(&build_batches(batches)); } #[test] @@ -323,14 +325,14 @@ fn test_sparse_deltas() { &["parquet", "B"], &["123", "B", "C"], ]; - run_parity_test(batches); + run_parity_test(&build_batches(batches)); } #[test] fn test_deltas_with_reset() { // Dictionary resets at ["C", "D"] let batches: &[&[&str]] = &[&["A"], &["A", "B"], &["C", "D"], &["A", "B", "C", "D"]]; - run_parity_test(batches); + run_parity_test(&build_batches(batches)); } /// FileWriter can only tolerate very specific patterns of delta dictionaries, @@ -338,7 +340,19 @@ fn test_deltas_with_reset() { #[test] fn test_deltas_with_file() { let batches: &[&[&str]] = &[&["A"], &["A", "B"], &["A", "B", "C"], &["A", "B", "C", "D"]]; - run_parity_test(batches); + run_parity_test(&build_batches(batches)); +} + +#[test] +fn test_deltas_with_in_struct() { + let batches: &[&[&str]] = &[&["A"], &["A", "B"], &["A", "B", "C"], &["A", "B", "C", "D"]]; + run_parity_test(&build_struct_batches(batches)); +} + +#[test] +fn test_deltas_with_in_list() { + let batches: &[&[&str]] = &[&["A"], &["A", "B"], &["A", "B", "C"], &["A", "B", "C", "D"]]; + run_parity_test(&build_list_batches(batches)); } /// Encode all batches three times and compare all three for the same results @@ -348,7 +362,7 @@ fn test_deltas_with_file() { /// - Stream encoding without delta /// - File encoding with delta (File format does not allow replacement /// dictionaries) -fn run_parity_test(batches: &[&[&str]]) { +fn run_parity_test(batches: &[RecordBatch]) { let delta_options = IpcWriteOptions::default().with_dictionary_handling(DictionaryHandling::Delta); let delta_stream_buf = write_all_to_stream(delta_options.clone(), batches); @@ -368,16 +382,16 @@ fn run_parity_test(batches: &[&[&str]]) { let (first_stream, other_streams) = streams.split_first_mut().unwrap(); for (idx, batch) in first_stream.by_ref().enumerate() { - let first_dict = extract_dictionary(batch); - let expected_values = batches[idx]; - assert_eq!(expected_values, &dict_to_vec(first_dict.clone())); + let first_dict = extract_dictionary(&batch); + let expected_values = dict_to_vec(&extract_dictionary(&batches[idx])); + assert_eq!(expected_values, dict_to_vec(&first_dict)); for stream in other_streams.iter_mut() { let next_batch = stream .next() .expect("All streams should yield same number of elements"); - let next_dict = extract_dictionary(next_batch); - assert_eq!(expected_values, &dict_to_vec(next_dict.clone())); + let next_dict = extract_dictionary(&next_batch); + assert_eq!(expected_values, dict_to_vec(&next_dict)); assert_eq!(first_dict, next_dict); } } @@ -390,7 +404,7 @@ fn run_parity_test(batches: &[&[&str]]) { } } -fn dict_to_vec(dict: DictionaryArray) -> Vec { +fn dict_to_vec(dict: &DictionaryArray) -> Vec { dict.downcast_dict::() .unwrap() .into_iter() @@ -418,35 +432,43 @@ fn get_file_batches(buf: Vec) -> Box> { ) } -fn extract_dictionary(batch: RecordBatch) -> DictionaryArray { - batch - .column(0) +fn extract_dictionary(batch: &RecordBatch) -> DictionaryArray { + let mut column = batch.column(0); + + // if we've been passed a struct, assume the first column contains the dict + if let Some(struct_arr) = column.as_any().downcast_ref::() { + column = struct_arr.column(0); + } + + // if we've been passed a list, assume the lists' values are the dict + if let Some(list_arr) = column.as_any().downcast_ref::() { + column = list_arr.values(); + } + + column .as_any() .downcast_ref::>() .unwrap() .clone() } -fn write_all_to_file(options: IpcWriteOptions, vals: &[&[&str]]) -> Vec { - let batches = build_batches(vals); +fn write_all_to_file(options: IpcWriteOptions, batches: &[RecordBatch]) -> Vec { let mut buf: Vec = Vec::new(); let mut writer = FileWriter::try_new_with_options(&mut buf, &batches[0].schema(), options).unwrap(); for batch in batches { - writer.write(&batch).unwrap(); + writer.write(batch).unwrap(); } writer.finish().unwrap(); buf } -fn write_all_to_stream(options: IpcWriteOptions, vals: &[&[&str]]) -> Vec { - let batches = build_batches(vals); - +fn write_all_to_stream(options: IpcWriteOptions, batches: &[RecordBatch]) -> Vec { let mut buf: Vec = Vec::new(); let mut writer = StreamWriter::try_new_with_options(&mut buf, &batches[0].schema(), options).unwrap(); for batch in batches { - writer.write(&batch).unwrap(); + writer.write(batch).unwrap(); } writer.finish().unwrap(); @@ -477,3 +499,71 @@ fn build_batch( RecordBatch::try_new(schema.clone(), vec![Arc::new(array) as ArrayRef]).unwrap() } + +/// build batches where the dictionary array is nested within a struct array. The dictionary array +/// is the first field within the struct. +fn build_struct_batches(vals: &[&[&str]]) -> Vec { + let total_vals = vals.iter().map(|v| v.len()).sum(); + let mut struct_builder = StructBuilder::from_fields( + vec![Field::new( + "struct", + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + false, + )], + total_vals, + ); + + vals.iter() + .map(|v| build_struct_batch(v, &mut struct_builder)) + .collect() +} + +fn build_struct_batch(vals: &[&str], struct_builder: &mut StructBuilder) -> RecordBatch { + for &val in vals { + let dict_builder = struct_builder + .field_builder::>(0) + .unwrap(); + dict_builder.append_value(val); + struct_builder.append(true); + } + + let array = struct_builder.finish_preserve_values(); + + let schema = Arc::new(Schema::new(vec![Field::new( + "dict", + array.data_type().clone(), + true, + )])); + + RecordBatch::try_new(schema.clone(), vec![Arc::new(array) as ArrayRef]).unwrap() +} + +/// builds batches where the dictionary array is nested within a list array +fn build_list_batches(vals: &[&[&str]]) -> Vec { + let mut list_builder = ListBuilder::new(StringDictionaryBuilder::::new()); + + vals.iter() + .map(|v| build_list_batch(v, &mut list_builder)) + .collect() +} + +fn build_list_batch( + vals: &[&str], + list_builder: &mut ListBuilder>, +) -> RecordBatch { + for &val in vals { + let vals_builder = list_builder.values(); + vals_builder.append(val).unwrap(); + list_builder.append(true); + } + + let array = list_builder.finish_preserve_values(); + + let schema = Arc::new(Schema::new(vec![Field::new( + "dict", + array.data_type().clone(), + true, + )])); + + RecordBatch::try_new(schema.clone(), vec![Arc::new(array) as ArrayRef]).unwrap() +} diff --git a/arrow-ipc/src/writer.rs b/arrow-ipc/src/writer.rs index a05072a2c47c..46e2dd7739e0 100644 --- a/arrow-ipc/src/writer.rs +++ b/arrow-ipc/src/writer.rs @@ -1351,7 +1351,7 @@ impl RecordBatchWriter for FileWriter { /// // You must set `.with_dictionary_handling(DictionaryHandling::Delta)` to /// // enable delta dictionaries in the writer /// let options = IpcWriteOptions::default().with_dictionary_handling(DictionaryHandling::Delta); -/// let mut writer = StreamWriter::try_new(&mut stream, &schema).unwrap(); +/// let mut writer = StreamWriter::try_new_with_options(&mut stream, &schema, options).unwrap(); /// /// // When writing the first batch, a dictionary message with 'a' and 'b' will be written /// // prior to the record batch. From 75f7916d2ae8d11320451f1c3e05680d69865d5f Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 4 May 2026 16:39:38 -0400 Subject: [PATCH 02/33] Prevent `FixedSizeBinaryArray` `i32` offset overflows (try 2) (#9872) # Which issue does this PR close? - Closes https://github.com/apache/arrow-rs/pull/9850 # Rationale for this change `FixedSizeBinaryArray::value_offset_at` works use `i32` arithmetic which can overflow. For offsets beyond `i32::MAX`, that can be bad # What changes are included in this PR? 1. Prevent any FixedSizedBinaryArrays from being constructed where the offset calculation could overflow 2. Add some other overflow checks As @adamreeve [pointed out](https://github.com/apache/arrow-rs/pull/9850#discussion_r3164949680) on https://github.com/apache/arrow-rs/pull/9850 there are several places where the `i32` arithmetic is problematic in `FixedSizeBinaryArray`. I will fix them for real in a different, follow on PR, by switching to entirely `usize` based arithmetic for offset calculations However, since I hope to backport this PR to older releases, I would like something that is easy to review and has the least potential for unintended consequences. # Are these changes tested? I added unit tests. However, I can't find any way to fully trigger the actual paths short of trying to allocate very large arrays, which I don't think is appropriate for unit tests. # Are there any user-facing changes? Better limit checking --- .../src/array/fixed_size_binary_array.rs | 120 ++++++++++++++++-- 1 file changed, 107 insertions(+), 13 deletions(-) diff --git a/arrow-array/src/array/fixed_size_binary_array.rs b/arrow-array/src/array/fixed_size_binary_array.rs index 928aa38a2727..3c3dc8f58920 100644 --- a/arrow-array/src/array/fixed_size_binary_array.rs +++ b/arrow-array/src/array/fixed_size_binary_array.rs @@ -108,7 +108,8 @@ impl FixedSizeBinaryArray { /// Create a new [`Scalar`] from `value` pub fn new_scalar(value: impl AsRef<[u8]>) -> Scalar { let v = value.as_ref(); - Scalar::new(Self::new(v.len() as _, Buffer::from(v), None)) + let size = i32::try_from(v.len()).expect("FixedSizeBinaryArray value length exceeds i32"); + Scalar::new(Self::new(size, Buffer::from(v), None)) } /// Create a new [`FixedSizeBinaryArray`] from the provided parts, returning an error on failure @@ -121,6 +122,7 @@ impl FixedSizeBinaryArray { /// * `size < 0` /// * `values.len() / size != nulls.len()` /// * `size == 0 && values.len() != 0` + /// * `len * size > i32::MAX` pub fn try_new( size: i32, values: Buffer, @@ -157,6 +159,8 @@ impl FixedSizeBinaryArray { } }; + Self::validate_lengths(s, len)?; + Ok(Self { data_type, value_data: values, @@ -166,6 +170,33 @@ impl FixedSizeBinaryArray { }) } + /// Some calculations below use i32 arithmetic which can overflow when + /// valid offsets are past i32::MAX. Until that is solved for real do not + /// permit constructing any FixedSizeBinaryArray that has a valid offset + /// past i32::MAX + fn validate_lengths(value_size: usize, len: usize) -> Result<(), ArrowError> { + // the offset is also calculated for the next element (i + 1) so + // check `len` (not last element index) to ensure that all offsets are valid + let max_offset = value_size.checked_mul(len).ok_or_else(|| { + ArrowError::InvalidArgumentError(format!( + "FixedSizeBinaryArray error: value size {value_size} * len {len} exceeds maximum valid offset" + )) + })?; + + let max_valid_offset: usize = i32::MAX.try_into().map_err(|_| { + ArrowError::InvalidArgumentError(format!( + "FixedSizeBinaryArray error: maximum valid offset exceeds i32::MAX, got {max_offset}" + )) + })?; + + if max_offset > max_valid_offset { + return Err(ArrowError::InvalidArgumentError(format!( + "FixedSizeBinaryArray error: value size {value_size} * length {len} exceeds maximum valid offset of {max_valid_offset}" + ))); + }; + Ok(()) + } + /// Create a new [`FixedSizeBinaryArray`] of length `len` where all values are null /// /// # Panics @@ -174,12 +205,17 @@ impl FixedSizeBinaryArray { /// /// * `size < 0` /// * `size * len` would overflow `usize` + /// * `size * len > i32::MAX` + /// * `size * len * 8` would overflow `usize` pub fn new_null(size: i32, len: usize) -> Self { const BITS_IN_A_BYTE: usize = 8; - let capacity_in_bytes = size.to_usize().unwrap().checked_mul(len).unwrap(); + let size_usize = size.to_usize().unwrap(); + Self::validate_lengths(size_usize, len).unwrap(); + let capacity_in_bytes = size_usize.checked_mul(len).unwrap(); + let capacity_in_bits = capacity_in_bytes.checked_mul(BITS_IN_A_BYTE).unwrap(); Self { data_type: DataType::FixedSizeBinary(size), - value_data: MutableBuffer::new_null(capacity_in_bytes * BITS_IN_A_BYTE).into(), + value_data: MutableBuffer::new_null(capacity_in_bits).into(), nulls: Some(NullBuffer::new_null(len)), value_length: size, len, @@ -347,8 +383,16 @@ impl FixedSizeBinaryArray { // Now that we know how large each element is we can reserve // sufficient capacity in the underlying mutable buffer for // the data. - buffer.reserve(iter_size_hint * len); - buffer.extend_zeros(slice.len() * prepend); + if let Some(capacity) = iter_size_hint.checked_mul(len) { + buffer.reserve(capacity); + } + let prepend_zeros = slice.len().checked_mul(prepend).ok_or_else(|| { + ArrowError::InvalidArgumentError(format!( + "FixedSizeBinaryArray error: value size {} * prepend {prepend} exceeds usize", + slice.len() + )) + })?; + buffer.extend_zeros(prepend_zeros); } bit_util::set_bit(null_buf.as_slice_mut(), len); buffer.extend_from_slice(slice); @@ -371,7 +415,13 @@ impl FixedSizeBinaryArray { let nulls = NullBuffer::from_unsliced_buffer(null_buf, len); - let size = size.unwrap_or(0) as i32; + let size = size.unwrap_or(0); + Self::validate_lengths(size, len)?; + let size = size.try_into().map_err(|_| { + ArrowError::InvalidArgumentError(format!( + "FixedSizeBinaryArray value length exceeds i32, got {size}" + )) + })?; Ok(Self { data_type: DataType::FixedSizeBinary(size), value_data: buffer.into(), @@ -410,12 +460,20 @@ impl FixedSizeBinaryArray { T: Iterator>, U: AsRef<[u8]>, { + let size_usize = size.to_usize().ok_or_else(|| { + ArrowError::InvalidArgumentError(format!("Size cannot be negative, got {size}")) + })?; let mut len = 0; let mut byte = 0; let iter_size_hint = iter.size_hint().0; let mut null_buf = MutableBuffer::new(bit_util::ceil(iter_size_hint, 8)); - let mut buffer = MutableBuffer::new(iter_size_hint * (size as usize)); + let capacity = iter_size_hint.checked_mul(size_usize).ok_or_else(|| { + ArrowError::InvalidArgumentError(format!( + "FixedSizeBinaryArray error: value size {size_usize} * len hint {iter_size_hint} exceeds usize" + )) + })?; + let mut buffer = MutableBuffer::new(capacity); iter.try_for_each(|item| -> Result<(), ArrowError> { // extend null bitmask by one byte per each 8 items @@ -427,7 +485,7 @@ impl FixedSizeBinaryArray { if let Some(slice) = item { let slice = slice.as_ref(); - if size as usize != slice.len() { + if size_usize != slice.len() { return Err(ArrowError::InvalidArgumentError(format!( "Nested array size mismatch: one is {}, and the other is {}", size, @@ -438,7 +496,7 @@ impl FixedSizeBinaryArray { bit_util::set_bit(null_buf.as_slice_mut(), len); buffer.extend_from_slice(slice); } else { - buffer.extend_zeros(size as usize); + buffer.extend_zeros(size_usize); } len += 1; @@ -447,6 +505,7 @@ impl FixedSizeBinaryArray { })?; let nulls = NullBuffer::from_unsliced_buffer(null_buf, len); + Self::validate_lengths(size_usize, len)?; Ok(Self { data_type: DataType::FixedSizeBinary(size), @@ -497,7 +556,9 @@ impl FixedSizeBinaryArray { } else { let len = slice.len(); size = Some(len); - buffer.reserve(iter_size_hint * len); + if let Some(capacity) = iter_size_hint.checked_mul(len) { + buffer.reserve(capacity); + } } buffer.extend_from_slice(slice); @@ -513,7 +574,13 @@ impl FixedSizeBinaryArray { )); } - let size = size.unwrap_or(0).try_into().unwrap(); + let size = size.unwrap_or(0); + Self::validate_lengths(size, len)?; + let size = size.try_into().map_err(|_| { + ArrowError::InvalidArgumentError(format!( + "FixedSizeBinaryArray value length exceeds i32, got {size}" + )) + })?; Ok(Self { data_type: DataType::FixedSizeBinary(size), value_data: buffer.into(), @@ -548,8 +615,15 @@ impl From for FixedSizeBinaryArray { _ => panic!("Expected data type to be FixedSizeBinary"), }; - let size = value_length as usize; - let value_data = buffers[0].slice_with_length(offset * size, len * size); + let size = value_length + .to_usize() + .expect("FixedSizeBinaryArray value length must be non-negative"); + Self::validate_lengths(size, len) + .expect("FixedSizeBinaryArray offsets must fit within i32"); + let value_data = buffers[0].slice_with_length( + offset.checked_mul(size).expect("offset overflow"), + len.checked_mul(size).expect("length overflow"), + ); Self { data_type, @@ -1040,6 +1114,26 @@ mod tests { array.value(4); } + #[test] + fn test_validate_lengths_allows_empty_array() { + FixedSizeBinaryArray::validate_lengths(1024, 0).unwrap(); + } + + #[test] + fn test_validate_lengths_allows_i32_max_offset() { + FixedSizeBinaryArray::validate_lengths(1, i32::MAX as usize).unwrap(); + FixedSizeBinaryArray::validate_lengths(262_176, 8191).unwrap(); + } + + #[test] + fn test_validate_lengths_rejects_offset_past_i32_max() { + let err = FixedSizeBinaryArray::validate_lengths(262_177, 8192).unwrap_err(); + assert_eq!( + err.to_string(), + "Invalid argument error: FixedSizeBinaryArray error: value size 262177 * length 8192 exceeds maximum valid offset of 2147483647", + ); + } + #[test] fn test_constructors() { let buffer = Buffer::from_vec(vec![0_u8; 10]); From 0e478d8f54b0a18735450b59d80ecc4dce13b012 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Mon, 4 May 2026 14:16:20 -0700 Subject: [PATCH 03/33] docs: Add guidance for AI assisted submissions to CONTRIBUTING.md (#9892) # Which issue does this PR close? None # Rationale for this change Arrow and Datafusion provide this guidance, it's high time we do as well. # What changes are included in this PR? Add links to the Arrow and ASF AI policies. # Are these changes tested? Docs only # Are there any user-facing changes? Only to project documentation --- CONTRIBUTING.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 83ee468c32e5..48075c5ef771 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -256,3 +256,21 @@ If sometimes you want to commit without checking, just run `git commit` with `-- ```bash git commit --no-verify -m "... commit message ..." ``` + +## AI Generated Submissions + +This project follows the guidance for AI generated submissions used by the +[Arrow Project](https://arrow.apache.org/docs/dev/developers/overview.html#ai-generated-code). +As such, it is expected that you will: + +- Only submit a PR if you are able to debug and own the changes yourself - review all generated + code to understand every detail +- Match the style and conventions used in the rest of the codebase, including PR titles and descriptions +- Be upfront about AI usage and summarise what was AI-generated +- If there are parts you don’t fully understand, leave comments on your own PR explaining what steps you took to verify correctness +- Watch for AI’s tendency to generate overly verbose comments, unnecessary test cases, and incorrect fixes +- Break down large PRs into smaller ones to make review easier + +It is also important for submitters to be aware of potential copyright issues. See the ASF's +[guidance](https://www.apache.org/legal/generative-tooling.html) on AI-generated code for further +information on licensing considerations. From f4a0f224a7bce30ca2cccb6ebcf13ebd3026173d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20Subiotto=20Marqu=C3=A9s?= Date: Tue, 5 May 2026 07:30:58 +0200 Subject: [PATCH 04/33] chore[benches]: add REE interleave benchmarks (#9849) In preparation for a specialized interleave kernel. # Which issue does this PR close? - No issue needed: benchmark addition # Rationale for this change I want to open a subsequent PR adding the specialized REE interleave kernel. # What changes are included in this PR? Benchmarks # Are these changes tested? Not necessary # Are there any user-facing changes? No Signed-off-by: Alfonso Subiotto Marques --- arrow/benches/interleave_kernels.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arrow/benches/interleave_kernels.rs b/arrow/benches/interleave_kernels.rs index 0688bb79adb1..4c4f5364ffe6 100644 --- a/arrow/benches/interleave_kernels.rs +++ b/arrow/benches/interleave_kernels.rs @@ -150,6 +150,14 @@ fn add_benchmark(c: &mut Criterion) { ) }; + let ree_run_ends = Int32Array::from_iter_values((1..=64).map(|i| i * 16)); + let ree_i64_values = create_primitive_array::(64, 0.0); + let ree_i64 = RunArray::::try_new(&ree_run_ends, &ree_i64_values).unwrap(); + + let dict_str_values = create_string_array_with_len::(20, 0.0, 12); + let dict_for_ree = create_dict_from_values::(64, 0.0, &dict_str_values); + let ree_dict = RunArray::::try_new(&ree_run_ends, &dict_for_ree).unwrap(); + let cases: &[(&str, &dyn Array)] = &[ ("i32(0.0)", &i32), ("i32(0.5)", &i32_opt), @@ -175,6 +183,8 @@ fn add_benchmark(c: &mut Criterion) { ("list_view(0.1,0.1,20)", &list_view_i64), ("list_view(0.0,0.0,20)", &list_view_i64_no_nulls), ("list_view_overlapping(80x,20)", &list_view_overlapping), + ("ree_i32(64 runs)", &ree_i64), + ("ree_i32>(64 runs)", &ree_dict), ]; for (prefix, base) in cases { From d88cc48b1d2ad6353d2705bf45918180aa2f844d Mon Sep 17 00:00:00 2001 From: RIchard Baah <137434454+Rich-T-kid@users.noreply.github.com> Date: Tue, 5 May 2026 09:50:49 -0400 Subject: [PATCH 05/33] replace Dictionary::try_new() calls with with_values. (#9894) # Which issue does this PR close? - Closes #9889. # Rationale for this change keeps consistency with the rest of the code base # What changes are included in this PR? replaced two try_new() calles with **with_values()**, logically the same. # Are these changes tested? yes, all test pass # Are there any user-facing changes? no --- arrow-cast/src/cast/dictionary.rs | 28 ++++++++++++---------------- arrow-string/src/substring.rs | 3 +-- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/arrow-cast/src/cast/dictionary.rs b/arrow-cast/src/cast/dictionary.rs index 0092bc0c87dd..601f50a4d001 100644 --- a/arrow-cast/src/cast/dictionary.rs +++ b/arrow-cast/src/cast/dictionary.rs @@ -361,22 +361,18 @@ where D: DecimalType + ArrowPrimitiveType, { let dict = pack_numeric_to_dictionary::(array, dict_value_type, cast_options)?; - let dict = dict - .as_dictionary::() - .downcast_dict::>() - .ok_or_else(|| { - ArrowError::ComputeError(format!( - "Internal Error: Cannot cast dict to {}Array", - D::PREFIX - )) - })?; - let value = dict.values().clone(); - // Set correct precision/scale - let value = value.with_precision_and_scale(precision, scale)?; - Ok(Arc::new(DictionaryArray::::try_new( - dict.keys().clone(), - Arc::new(value), - )?)) + let dict = dict.as_dictionary::(); + let typed = dict.downcast_dict::>().ok_or_else(|| { + ArrowError::ComputeError(format!( + "Internal Error: Cannot cast dict to {}Array", + D::PREFIX + )) + })?; + let value = typed + .values() + .clone() + .with_precision_and_scale(precision, scale)?; + Ok(Arc::new(dict.with_values(Arc::new(value)))) } pub(crate) fn string_view_to_dictionary( diff --git a/arrow-string/src/substring.rs b/arrow-string/src/substring.rs index 05b3888a444a..3dc3bd695c44 100644 --- a/arrow-string/src/substring.rs +++ b/arrow-string/src/substring.rs @@ -86,8 +86,7 @@ pub fn substring( stringify!($gt), array.data_type()) }); let values = substring(dict.values(), start, length)?; - let result = DictionaryArray::try_new(dict.keys().clone(), values)?; - Ok(Arc::new(result)) + Ok(Arc::new(dict.with_values(values))) }, )* t => panic!("Unsupported dictionary key type: {}", t) From 99998d63086bd1d338139091aee38b850b07efb5 Mon Sep 17 00:00:00 2001 From: RIchard Baah <137434454+Rich-T-kid@users.noreply.github.com> Date: Tue, 5 May 2026 10:28:23 -0400 Subject: [PATCH 06/33] API to help with the pattern of 'replaces the values of the REE array (#9891) # Which issue does this PR close? - Closes #9854. # Rationale for this change over time the pattern of replacing REE values without creating a new instance has emerged, buts approached have been scatered. this PR attempts to consolidate them into a single place. # What changes are included in this PR? Includes updating call sites that used to manually replace REE values by constructing new ones, switching them to use a macro instead. # Are these changes tested? yes # Are there any user-facing changes? no --- arrow-arith/src/temporal.rs | 39 ++++++++++++++++++++ arrow-array/src/array/run_array.rs | 59 ++++++++++++++++++++++++++++++ arrow-string/src/length.rs | 57 +++++++++++++++++++---------- 3 files changed, 136 insertions(+), 19 deletions(-) diff --git a/arrow-arith/src/temporal.rs b/arrow-arith/src/temporal.rs index 12970f4498fa..869fba29cd15 100644 --- a/arrow-arith/src/temporal.rs +++ b/arrow-arith/src/temporal.rs @@ -23,6 +23,7 @@ use arrow_array::cast::AsArray; use cast::as_primitive_array; use chrono::{Datelike, TimeZone, Timelike, Utc}; +use arrow_array::ree_map; use arrow_array::temporal_conversions::{ MICROSECONDS, MICROSECONDS_IN_DAY, MILLISECONDS, MILLISECONDS_IN_DAY, NANOSECONDS, NANOSECONDS_IN_DAY, SECONDS_IN_DAY, date32_to_datetime, date64_to_datetime, @@ -194,6 +195,15 @@ pub fn date_part(array: &dyn Array, part: DatePart) -> Result match k.data_type() { + DataType::Int16 => ree_map!(array, Int16Type, |a| date_part(a, part)), + DataType::Int32 => ree_map!(array, Int32Type, |a| date_part(a, part)), + DataType::Int64 => ree_map!(array, Int64Type, |a| date_part(a, part)), + _ => Err(ArrowError::InvalidArgumentError(format!( + "Invalid run-end type: {:?}", + k.data_type() + ))), + }, t => return_compute_error_with!(format!("{part} does not support"), t), ) } @@ -2040,4 +2050,33 @@ mod tests { assert_eq!(2015, actual.value(1)); assert_eq!(2016, actual.value(2)); } + + #[test] + fn test_ree_timestamp_year() { + let vals: TimestampSecondArray = + vec![Some(1514764800), Some(1550636625), Some(1550636625)].into(); + let run_ends = Int32Array::from(vec![1, 2, 3]); + let ree = RunArray::try_new(&run_ends, &vals).unwrap(); + + let b = date_part(&ree, DatePart::Year).unwrap(); + let ree_result = b.as_run_opt::().unwrap(); + let values = ree_result.values().as_primitive::(); + assert_eq!(2018, values.value(0)); + assert_eq!(2019, values.value(1)); + assert_eq!(2019, values.value(2)); + } + + #[test] + fn test_ree_date64_month() { + let vals: PrimitiveArray = + vec![Some(1514764800000), Some(1550636625000)].into(); + let run_ends = Int64Array::from(vec![2, 4]); + let ree = RunArray::try_new(&run_ends, &vals).unwrap(); + + let b = date_part(&ree, DatePart::Month).unwrap(); + let ree_result = b.as_run_opt::().unwrap(); + let values = ree_result.values().as_primitive::(); + assert_eq!(1, values.value(0)); + assert_eq!(2, values.value(1)); + } } diff --git a/arrow-array/src/array/run_array.rs b/arrow-array/src/array/run_array.rs index f317af6a10f0..02bc730b32df 100644 --- a/arrow-array/src/array/run_array.rs +++ b/arrow-array/src/array/run_array.rs @@ -30,6 +30,25 @@ use crate::{ types::{Int16Type, Int32Type, Int64Type, RunEndIndexType}, }; +/// Recursively applies a function to the values of a RunEndEncoded array, preserving the run structure. +/// +/// # Example +/// +/// ```ignore +/// let result = ree_recurse!(array, Int32Type, my_function)?; +/// ``` +/// +/// This macro is useful for implementing functions that should work on the logical values +/// of a REE array while preserving the run-end encoding structure. +#[macro_export] +macro_rules! ree_map { + ($array:expr, $run_type:ty, $func:expr) => {{ + let ree = $array.as_run_opt::<$run_type>().unwrap(); + let inner_values = $func(ree.values().as_ref())?; + Ok(std::sync::Arc::new(ree.with_values(inner_values))) + }}; +} + /// An array of [run-end encoded values]. /// /// This encoding is variation on [run-length encoding (RLE)] and is good for representing @@ -200,6 +219,46 @@ impl RunArray { &self.values } + /// Returns a new [`RunArray`] with the same `run_ends` and the supplied `values`. + /// + /// # Panics + /// + /// Panics if `values.len()` does not equal `self.values().len()`. + /// + /// # Example + /// + /// ``` + /// # use std::sync::Arc; + /// # use arrow_array::{RunArray, Int32Array, StringArray, ArrayRef,Array}; + /// # use arrow_array::types::Int32Type; + /// // A RunArray logically representing ["a", "a", "b", "c", "c"] + /// let run_ends = Int32Array::from(vec![2, 3, 5]); + /// let values: ArrayRef = Arc::new(StringArray::from(vec!["a", "b", "c"])); + /// let run_array = RunArray::::try_new(&run_ends, &values).unwrap(); + /// + /// // Swap in new values while keeping the same run pattern. + /// // The result logically represents ["x", "x", "y", "z", "z"]. + /// let new_values: ArrayRef = Arc::new(StringArray::from(vec!["x", "y", "z"])); + /// let new_run_array = run_array.with_values(new_values); + /// + /// assert_eq!(new_run_array.len(), 5); + /// assert_eq!(new_run_array.run_ends().values(), &[2, 3, 5]); + /// ``` + pub fn with_values(&self, values: ArrayRef) -> Self { + assert_eq!(values.len(), self.values().len()); + let (run_ends_field, values_field) = match &self.data_type { + DataType::RunEndEncoded(r, v) => (r, v), + _ => unreachable!("RunArray should have type RunEndEncoded"), + }; + let data_type = + DataType::RunEndEncoded(Arc::clone(run_ends_field), Arc::clone(values_field)); + Self { + data_type, + run_ends: self.run_ends.clone(), + values, + } + } + /// Similar to [`values`] but accounts for logical slicing, returning only the values /// that are part of the logical slice of this array. /// diff --git a/arrow-string/src/length.rs b/arrow-string/src/length.rs index ff98c5632b74..feefe1247e2c 100644 --- a/arrow-string/src/length.rs +++ b/arrow-string/src/length.rs @@ -17,25 +17,12 @@ //! Defines kernel for length of string arrays and binary arrays +use arrow_array::ree_map; use arrow_array::*; use arrow_array::{cast::AsArray, types::*}; use arrow_buffer::{ArrowNativeType, NullBuffer, OffsetBuffer}; use arrow_schema::{ArrowError, DataType}; use std::sync::Arc; -macro_rules! ree_length { - ($array:expr, $run_type:ty, $k:expr, $v:expr) => {{ - let ree = $array.as_run_opt::<$run_type>().unwrap(); - let inner_value_lengths = length(ree.values().as_ref())?; - let out_ree = unsafe { - RunArray::<$run_type>::new_unchecked( - DataType::RunEndEncoded(Arc::clone($k), Arc::clone($v)), - ree.run_ends().clone(), - inner_value_lengths, - ) - }; - Ok(Arc::new(out_ree) as ArrayRef) - }}; -} fn length_impl( offsets: &OffsetBuffer, @@ -130,10 +117,10 @@ pub fn length(array: &dyn Array) -> Result { list.nulls().cloned(), )?)) } - DataType::RunEndEncoded(k, v) => match k.data_type() { - DataType::Int16 => ree_length!(array, Int16Type, &k, &v), - DataType::Int32 => ree_length!(array, Int32Type, &k, &v), - DataType::Int64 => ree_length!(array, Int64Type, &k, &v), + DataType::RunEndEncoded(k, _) => match k.data_type() { + DataType::Int16 => ree_map!(array, Int16Type, length), + DataType::Int32 => ree_map!(array, Int32Type, length), + DataType::Int64 => ree_map!(array, Int64Type, length), _ => Err(ArrowError::InvalidArgumentError(format!( "Invalid run-end type: {:?}", k.data_type() @@ -149,7 +136,7 @@ pub fn length(array: &dyn Array) -> Result { /// /// * this only accepts StringArray/Utf8, LargeString/LargeUtf8, StringViewArray/Utf8View, /// BinaryArray, LargeBinaryArray, BinaryViewArray, and FixedSizeBinaryArray, -/// or DictionaryArray with above Arrays as values +/// or DictionaryArray/REE with above Arrays as values /// * bit_length of null is null. /// * bit_length is in number of bits pub fn bit_length(array: &dyn Array) -> Result { @@ -203,6 +190,15 @@ pub fn bit_length(array: &dyn Array) -> Result { array.nulls().cloned(), )?)) } + DataType::RunEndEncoded(k, _) => match k.data_type() { + DataType::Int16 => ree_map!(array, Int16Type, bit_length), + DataType::Int32 => ree_map!(array, Int32Type, bit_length), + DataType::Int64 => ree_map!(array, Int64Type, bit_length), + _ => Err(ArrowError::InvalidArgumentError(format!( + "Invalid run-end type: {:?}", + k.data_type() + ))), + }, other => Err(ArrowError::ComputeError(format!( "bit_length not supported for {other:?}" ))), @@ -903,4 +899,27 @@ mod tests { assert!(length(&ree_array).is_err()); } + + #[test] + fn bit_length_test_ree_utf8() { + use arrow_array::RunArray; + use arrow_array::types::Int32Type; + + let strings = StringArray::from(vec!["hello", "world", "test"]); + let run_ends = PrimitiveArray::::from(vec![1i32, 2, 3]); + let ree_array = RunArray::::try_new(&run_ends, &strings).unwrap(); + + let result = bit_length(&ree_array).unwrap(); + let result_values = result + .as_any() + .downcast_ref::>() + .unwrap() + .values() + .as_any() + .downcast_ref::() + .unwrap(); + + let expected: Int32Array = vec![40, 40, 32].into(); + assert_eq!(&expected, result_values); + } } From 21f739cb5f2f23629fa0a0aad2aaf7b1d6f6e81c Mon Sep 17 00:00:00 2001 From: Hippolyte Barraud Date: Tue, 5 May 2026 13:16:07 -0400 Subject: [PATCH 07/33] test(parquet): replace `InMemoryArrayReader` with `PrimitiveArrayReader` in tests (#9847) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Which issue does this PR close? - Contributes to #9731 - Dependency of #9848 # Rationale for this change `InMemoryArrayReader` couples a column's in-memory representation (an Arrow array) with its storage representation (def/rep levels) and assumes a 1:1 mapping between array elements and levels. This holds when list readers consume fully-padded child arrays — one element per level, nulls included. Upcoming work (see #9848) pushes null filtering from `ListArrayReader` down into the child reader at the storage level, breaking that 1:1 assumption: the child returns fewer array elements than levels, and the mapping between them depends on the filtering logic itself. Keeping the mock would mean reimplementing that logic: testing filtered output against a second, hand-rolled filter. # What changes are included in this PR? Replace `InMemoryArrayReader` with real `PrimitiveArrayReader` instances backed by in-memory Parquet pages. Tests now accept raw non-null values and levels (matching what Parquet actually stores) and exercise the production `RecordReader` path. # Are these changes tested? This is a net positive in test coverage. The existing tests now exercise real readers instead of in-memory and already-dense representations. # Are there any user-facing changes? None. Signed-off-by: Hippolyte Barraud --- .../array_reader/fixed_size_list_array.rs | 117 ++++---------- parquet/src/arrow/array_reader/list_array.rs | 94 +++-------- .../src/arrow/array_reader/list_view_array.rs | 72 ++------- .../src/arrow/array_reader/struct_array.rs | 51 ++---- parquet/src/arrow/array_reader/test_util.rs | 148 +++++------------- parquet/src/arrow/arrow_reader/read_plan.rs | 10 +- 6 files changed, 128 insertions(+), 364 deletions(-) diff --git a/parquet/src/arrow/array_reader/fixed_size_list_array.rs b/parquet/src/arrow/array_reader/fixed_size_list_array.rs index 8ef3bd6c2a4b..518cd8625ed5 100644 --- a/parquet/src/arrow/array_reader/fixed_size_list_array.rs +++ b/parquet/src/arrow/array_reader/fixed_size_list_array.rs @@ -226,7 +226,7 @@ mod tests { use super::*; use crate::arrow::{ ArrowWriter, - array_reader::{ListArrayReader, test_util::InMemoryArrayReader}, + array_reader::{ListArrayReader, test_util::make_int32_page_reader}, arrow_reader::{ArrowReaderBuilder, ArrowReaderOptions, ParquetRecordBatchReader}, }; use arrow::datatypes::{Field, Int32Type}; @@ -254,28 +254,16 @@ mod tests { 3, ); - let array = Arc::new(PrimitiveArray::::from(vec![ - None, - Some(1), - None, - Some(2), - None, - Some(3), - Some(4), - Some(5), - None, - None, - None, - ])); - let item_array_reader = InMemoryArrayReader::new( - ArrowType::Int32, - array, - Some(vec![0, 3, 2, 3, 0, 3, 3, 3, 2, 2, 2]), - Some(vec![0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1]), + let item_array_reader = make_int32_page_reader( + &[1, 2, 3, 4, 5], + &[0, 3, 2, 3, 0, 3, 3, 3, 2, 2, 2], + &[0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1], + 3, + 1, ); let mut list_array_reader = FixedSizeListArrayReader::new( - Box::new(item_array_reader), + item_array_reader, 3, ArrowType::FixedSizeList(Arc::new(Field::new_list_field(ArrowType::Int32, true)), 3), 2, @@ -303,25 +291,16 @@ mod tests { 2, ); - let array = Arc::new(PrimitiveArray::::from(vec![ - Some(1), - None, - Some(2), - Some(3), - None, - None, - Some(4), - Some(5), - ])); - let item_array_reader = InMemoryArrayReader::new( - ArrowType::Int32, - array, - Some(vec![2, 1, 2, 2, 1, 1, 2, 2]), - Some(vec![0, 1, 0, 1, 0, 1, 0, 1]), + let item_array_reader = make_int32_page_reader( + &[1, 2, 3, 4, 5], + &[2, 1, 2, 2, 1, 1, 2, 2], + &[0, 1, 0, 1, 0, 1, 0, 1], + 2, + 1, ); let mut list_array_reader = FixedSizeListArrayReader::new( - Box::new(item_array_reader), + item_array_reader, 2, ArrowType::FixedSizeList(Arc::new(Field::new_list_field(ArrowType::Int32, true)), 2), 1, @@ -381,28 +360,15 @@ mod tests { let expected = FixedSizeListArray::from(l1); - let values = Arc::new(PrimitiveArray::::from(vec![ - None, - Some(1), - Some(2), - None, - Some(3), - None, - Some(4), - Some(5), - None, - None, - ])); - - let item_array_reader = InMemoryArrayReader::new( - ArrowType::Int32, - values, - Some(vec![0, 5, 5, 4, 5, 0, 5, 5, 4, 4]), - Some(vec![0, 0, 2, 0, 2, 0, 0, 2, 0, 2]), + let item_array_reader = make_int32_page_reader( + &[1, 2, 3, 4, 5], + &[0, 5, 5, 4, 5, 0, 5, 5, 4, 4], + &[0, 0, 2, 0, 2, 0, 0, 2, 0, 2], + 5, + 2, ); - let l2 = - FixedSizeListArrayReader::new(Box::new(item_array_reader), 2, l2_type, 4, 2, false); + let l2 = FixedSizeListArrayReader::new(item_array_reader, 2, l2_type, 4, 2, false); let mut l1 = FixedSizeListArrayReader::new(Box::new(l2), 1, l1_type, 3, 1, true); let expected_1 = expected.slice(0, 2); @@ -423,18 +389,10 @@ mod tests { 0, ); - let array = Arc::new(PrimitiveArray::::from(vec![ - None, None, None, None, - ])); - let item_array_reader = InMemoryArrayReader::new( - ArrowType::Int32, - array, - Some(vec![0, 1, 0, 1]), - Some(vec![0, 0, 0, 0]), - ); + let item_array_reader = make_int32_page_reader(&[], &[0, 1, 0, 1], &[0, 0, 0, 0], 2, 1); let mut list_array_reader = FixedSizeListArrayReader::new( - Box::new(item_array_reader), + item_array_reader, 0, ArrowType::FixedSizeList(Arc::new(Field::new_list_field(ArrowType::Int32, true)), 0), 2, @@ -467,33 +425,20 @@ mod tests { builder.append(false); let expected = builder.finish(); - let array = Arc::new(PrimitiveArray::::from(vec![ - Some(1), - None, - Some(3), - None, - Some(4), - None, - Some(5), - Some(6), - None, - None, - None, - ])); - let inner_type = ArrowType::List(Arc::new(Field::new_list_field(ArrowType::Int32, true))); let list_type = ArrowType::FixedSizeList(Arc::new(Field::new_list_field(inner_type.clone(), true)), 2); - let item_array_reader = InMemoryArrayReader::new( - ArrowType::Int32, - array, - Some(vec![5, 4, 5, 2, 5, 3, 5, 5, 4, 4, 0]), - Some(vec![0, 2, 2, 1, 0, 1, 0, 2, 1, 2, 0]), + let item_array_reader = make_int32_page_reader( + &[1, 3, 4, 5, 6], + &[5, 4, 5, 2, 5, 3, 5, 5, 4, 4, 0], + &[0, 2, 2, 1, 0, 1, 0, 2, 1, 2, 0], + 5, + 2, ); let inner_array_reader = - ListArrayReader::::new(Box::new(item_array_reader), inner_type, 4, 2, true); + ListArrayReader::::new(item_array_reader, inner_type, 4, 2, true); let mut list_array_reader = FixedSizeListArrayReader::new(Box::new(inner_array_reader), 2, list_type, 2, 1, true); diff --git a/parquet/src/arrow/array_reader/list_array.rs b/parquet/src/arrow/array_reader/list_array.rs index a7adc01b912d..e6c834096902 100644 --- a/parquet/src/arrow/array_reader/list_array.rs +++ b/parquet/src/arrow/array_reader/list_array.rs @@ -248,7 +248,7 @@ mod tests { use super::*; use crate::arrow::array_reader::ArrayReaderBuilder; use crate::arrow::array_reader::list_array::ListArrayReader; - use crate::arrow::array_reader::test_util::InMemoryArrayReader; + use crate::arrow::array_reader::test_util::make_int32_page_reader; use crate::arrow::arrow_reader::DEFAULT_BATCH_SIZE; use crate::arrow::arrow_reader::metrics::ArrowReaderMetrics; use crate::arrow::schema::parquet_to_arrow_schema_and_fields; @@ -257,7 +257,7 @@ mod tests { use crate::file::reader::{FileReader, SerializedFileReader}; use crate::schema::parser::parse_message_type; use crate::schema::types::SchemaDescriptor; - use arrow::datatypes::{Field, Int32Type as ArrowInt32, Int32Type}; + use arrow::datatypes::{Field, Int32Type}; use arrow_array::{Array, PrimitiveArray}; use arrow_data::ArrayDataBuilder; use arrow_schema::Fields; @@ -354,36 +354,15 @@ mod tests { let expected = GenericListArray::::from(l1); - let values = Arc::new(PrimitiveArray::::from(vec![ - Some(1), - None, - None, - Some(4), - None, - None, - Some(7), - None, - Some(1), - Some(2), - Some(3), - Some(4), - None, - Some(6), - None, - None, - None, - Some(11), - ])); - - let item_array_reader = InMemoryArrayReader::new( - ArrowType::Int32, - values, - Some(vec![6, 5, 3, 6, 4, 2, 6, 4, 6, 6, 6, 6, 5, 6, 3, 0, 1, 6]), - Some(vec![0, 3, 2, 2, 2, 1, 1, 1, 1, 3, 3, 2, 3, 3, 2, 0, 0, 0]), + let item_array_reader = make_int32_page_reader( + &[1, 4, 7, 1, 2, 3, 4, 6, 11], + &[6, 5, 3, 6, 4, 2, 6, 4, 6, 6, 6, 6, 5, 6, 3, 0, 1, 6], + &[0, 3, 2, 2, 2, 1, 1, 1, 1, 3, 3, 2, 3, 3, 2, 0, 0, 0], + 6, + 3, ); - let l3 = - ListArrayReader::::new(Box::new(item_array_reader), l3_type, 5, 3, true); + let l3 = ListArrayReader::::new(item_array_reader, l3_type, 5, 3, true); let l2 = ListArrayReader::::new(Box::new(l3), l2_type, 3, 2, false); @@ -411,28 +390,16 @@ mod tests { Some(vec![None, Some(1)]), ]); - let array = Arc::new(PrimitiveArray::::from(vec![ - Some(1), - None, - Some(2), - None, - Some(3), - Some(4), - None, - None, - None, - Some(1), - ])); - - let item_array_reader = InMemoryArrayReader::new( - ArrowType::Int32, - array, - Some(vec![2, 1, 2, 0, 2, 2, 0, 0, 1, 2]), - Some(vec![0, 1, 1, 0, 0, 1, 0, 0, 0, 1]), + let item_array_reader = make_int32_page_reader( + &[1, 2, 3, 4, 1], + &[2, 1, 2, 0, 2, 2, 0, 0, 1, 2], + &[0, 1, 1, 0, 0, 1, 0, 0, 0, 1], + 2, + 1, ); let mut list_array_reader = ListArrayReader::::new( - Box::new(item_array_reader), + item_array_reader, list_type::(ArrowType::Int32, true), 1, 1, @@ -460,31 +427,16 @@ mod tests { Some(vec![None, Some(1)]), ]); - let array = Arc::new(PrimitiveArray::::from(vec![ - Some(1), - None, - Some(2), - None, - None, - Some(3), - Some(4), - None, - None, - None, - None, - None, - Some(1), - ])); - - let item_array_reader = InMemoryArrayReader::new( - ArrowType::Int32, - array, - Some(vec![3, 2, 3, 0, 1, 3, 3, 1, 1, 0, 1, 2, 3]), - Some(vec![0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1]), + let item_array_reader = make_int32_page_reader( + &[1, 2, 3, 4, 1], + &[3, 2, 3, 0, 1, 3, 3, 1, 1, 0, 1, 2, 3], + &[0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1], + 3, + 1, ); let mut list_array_reader = ListArrayReader::::new( - Box::new(item_array_reader), + item_array_reader, list_type::(ArrowType::Int32, true), 2, 1, diff --git a/parquet/src/arrow/array_reader/list_view_array.rs b/parquet/src/arrow/array_reader/list_view_array.rs index 294135d41f8b..357ab9dc14ae 100644 --- a/parquet/src/arrow/array_reader/list_view_array.rs +++ b/parquet/src/arrow/array_reader/list_view_array.rs @@ -106,9 +106,8 @@ impl ArrayReader for ListViewArrayReader() { // [[1, null, 2], null, [], [3, 4], [], [], null, [], [null, 1]] @@ -125,27 +124,12 @@ mod tests { Some(vec![None, Some(1)]), ]); - let array = Arc::new(PrimitiveArray::::from(vec![ - Some(1), - None, - Some(2), - None, - None, - Some(3), - Some(4), - None, - None, - None, - None, - None, - Some(1), - ])); - - let item_array_reader = InMemoryArrayReader::new( - ArrowType::Int32, - array, - Some(vec![3, 2, 3, 0, 1, 3, 3, 1, 1, 0, 1, 2, 3]), - Some(vec![0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1]), + let item_array_reader = make_int32_page_reader( + &[1, 2, 3, 4, 1], + &[3, 2, 3, 0, 1, 3, 3, 1, 1, 0, 1, 2, 3], + &[0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1], + 3, + 1, ); let field = Arc::new(arrow_schema::Field::new_list_field(ArrowType::Int32, true)); @@ -155,13 +139,8 @@ mod tests { ArrowType::ListView(field) }; - let mut list_view_array_reader = ListViewArrayReader::::new( - Box::new(item_array_reader), - data_type, - 2, - 1, - true, - ); + let mut list_view_array_reader = + ListViewArrayReader::::new(item_array_reader, data_type, 2, 1, true); let actual = list_view_array_reader.next_batch(1024).unwrap(); let actual = actual @@ -184,24 +163,12 @@ mod tests { Some(vec![None, Some(1)]), ]); - let array = Arc::new(PrimitiveArray::::from(vec![ - Some(1), - None, - Some(2), - None, - Some(3), - Some(4), - None, - None, - None, - Some(1), - ])); - - let item_array_reader = InMemoryArrayReader::new( - ArrowType::Int32, - array, - Some(vec![2, 1, 2, 0, 2, 2, 0, 0, 1, 2]), - Some(vec![0, 1, 1, 0, 0, 1, 0, 0, 0, 1]), + let item_array_reader = make_int32_page_reader( + &[1, 2, 3, 4, 1], + &[2, 1, 2, 0, 2, 2, 0, 0, 1, 2], + &[0, 1, 1, 0, 0, 1, 0, 0, 0, 1], + 2, + 1, ); let field = Arc::new(arrow_schema::Field::new_list_field(ArrowType::Int32, true)); @@ -211,13 +178,8 @@ mod tests { ArrowType::ListView(field) }; - let mut list_view_array_reader = ListViewArrayReader::::new( - Box::new(item_array_reader), - data_type, - 1, - 1, - false, - ); + let mut list_view_array_reader = + ListViewArrayReader::::new(item_array_reader, data_type, 1, 1, false); let actual = list_view_array_reader.next_batch(1024).unwrap(); let actual = actual diff --git a/parquet/src/arrow/array_reader/struct_array.rs b/parquet/src/arrow/array_reader/struct_array.rs index 63b20eb0d655..da92d410f32b 100644 --- a/parquet/src/arrow/array_reader/struct_array.rs +++ b/parquet/src/arrow/array_reader/struct_array.rs @@ -224,39 +224,27 @@ impl ArrayReader for StructArrayReader { mod tests { use super::*; use crate::arrow::array_reader::ListArrayReader; - use crate::arrow::array_reader::test_util::InMemoryArrayReader; + use crate::arrow::array_reader::test_util::make_int32_page_reader; use arrow::buffer::Buffer; use arrow::datatypes::Field; use arrow_array::cast::AsArray; - use arrow_array::{Array, Int32Array, ListArray}; + use arrow_array::{Array, ListArray}; use arrow_schema::Fields; #[test] fn test_struct_array_reader() { - let array_1 = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])); - let array_reader_1 = InMemoryArrayReader::new( - ArrowType::Int32, - array_1.clone(), - Some(vec![0, 1, 2, 3, 1]), - Some(vec![0, 1, 1, 1, 1]), - ); + let array_reader_1 = make_int32_page_reader(&[4], &[0, 1, 2, 3, 1], &[0, 1, 1, 1, 1], 3, 1); - let array_2 = Arc::new(Int32Array::from(vec![5, 4, 3, 2, 1])); - let array_reader_2 = InMemoryArrayReader::new( - ArrowType::Int32, - array_2.clone(), - Some(vec![0, 1, 3, 1, 2]), - Some(vec![0, 1, 1, 1, 1]), - ); + let array_reader_2 = make_int32_page_reader(&[3], &[0, 1, 3, 1, 2], &[0, 1, 1, 1, 1], 3, 1); let struct_type = ArrowType::Struct(Fields::from(vec![ - Field::new("f1", array_1.data_type().clone(), true), - Field::new("f2", array_2.data_type().clone(), true), + Field::new("f1", ArrowType::Int32, true), + Field::new("f2", ArrowType::Int32, true), ])); let mut struct_array_reader = StructArrayReader::new( struct_type, - vec![Box::new(array_reader_1), Box::new(array_reader_2)], + vec![array_reader_1, array_reader_2], 1, 1, true, @@ -306,28 +294,11 @@ mod tests { )]; let expected = StructArray::from((struct_fields, validity)); - let array = Arc::new(Int32Array::from_iter(vec![ - Some(1), - Some(2), - None, - None, - None, - None, - ])); - let reader = InMemoryArrayReader::new( - ArrowType::Int32, - array, - Some(vec![4, 4, 3, 2, 1, 0]), - Some(vec![0, 1, 1, 0, 0, 0]), - ); + let reader = + make_int32_page_reader(&[1, 2], &[4, 4, 3, 2, 1, 0], &[0, 1, 1, 0, 0, 0], 4, 1); - let list_reader = ListArrayReader::::new( - Box::new(reader), - expected_l.data_type().clone(), - 3, - 1, - true, - ); + let list_reader = + ListArrayReader::::new(reader, expected_l.data_type().clone(), 3, 1, true); let mut struct_reader = StructArrayReader::new( expected.data_type().clone(), diff --git a/parquet/src/arrow/array_reader/test_util.rs b/parquet/src/arrow/array_reader/test_util.rs index 45af5a2777b3..afba7c2c30c2 100644 --- a/parquet/src/arrow/array_reader/test_util.rs +++ b/parquet/src/arrow/array_reader/test_util.rs @@ -15,16 +15,13 @@ // specific language governing permissions and limitations // under the License. -use arrow_array::{Array, ArrayRef}; -use arrow_schema::DataType as ArrowType; use bytes::Bytes; -use std::any::Any; use std::sync::Arc; use crate::arrow::array_reader::ArrayReader; use crate::basic::{ConvertedType, Encoding, Type as PhysicalType}; use crate::column::page::{PageIterator, PageReader}; -use crate::data_type::{ByteArray, ByteArrayType}; +use crate::data_type::{ByteArray, ByteArrayType, Int32Type}; use crate::encodings::encoding::{DictEncoder, Encoder, get_encoder}; use crate::errors::Result; use crate::schema::types::{ColumnDescPtr, ColumnDescriptor, ColumnPath, Type}; @@ -91,112 +88,51 @@ pub fn byte_array_all_encodings( (pages, encoded_dictionary) } -/// Array reader for test. -pub struct InMemoryArrayReader { - data_type: ArrowType, - array: ArrayRef, - def_levels: Option>, - rep_levels: Option>, - last_idx: usize, - cur_idx: usize, - need_consume_records: usize, -} - -impl InMemoryArrayReader { - pub fn new( - data_type: ArrowType, - array: ArrayRef, - def_levels: Option>, - rep_levels: Option>, - ) -> Self { - assert!( - def_levels - .as_ref() - .map(|d| d.len() == array.len()) - .unwrap_or(true) - ); - - assert!( - rep_levels - .as_ref() - .map(|r| r.len() == array.len()) - .unwrap_or(true) - ); - - Self { - data_type, - array, - def_levels, - rep_levels, - cur_idx: 0, - last_idx: 0, - need_consume_records: 0, - } - } -} - -impl ArrayReader for InMemoryArrayReader { - fn as_any(&self) -> &dyn Any { - self - } - - fn get_data_type(&self) -> &ArrowType { - &self.data_type - } - - fn read_records(&mut self, batch_size: usize) -> Result { - assert_ne!(batch_size, 0); - // This replicates the logical normally performed by - // RecordReader to delimit semantic records - let read = match &self.rep_levels { - Some(rep_levels) => { - let rep_levels = &rep_levels[self.cur_idx..]; - let mut levels_read = 0; - let mut records_read = 0; - while levels_read < rep_levels.len() && records_read < batch_size { - if rep_levels[levels_read] == 0 { - records_read += 1; // Start of new record - } - levels_read += 1; - } - - // Find end of current record - while levels_read < rep_levels.len() && rep_levels[levels_read] != 0 { - levels_read += 1 - } - levels_read - } - None => batch_size.min(self.array.len() - self.cur_idx), - }; - self.need_consume_records += read; - Ok(read) - } - - fn consume_batch(&mut self) -> Result { - let batch_size = self.need_consume_records; - assert_ne!(batch_size, 0); - self.last_idx = self.cur_idx; - self.cur_idx += batch_size; - self.need_consume_records = 0; - Ok(self.array.slice(self.last_idx, batch_size)) - } +/// Build a real `PrimitiveArrayReader` from raw non-null values +/// and definition/repetition levels. This exercises the full production +/// `RecordReader` code path (including selective padding when a parent +/// `ListArrayReader` calls `enable_selective_padding`). +/// +/// `values` must contain only the non-null values (entries where +/// `def_levels[i] == max_def_level`), in order, as Parquet encodes them. +pub fn make_int32_page_reader( + values: &[i32], + def_levels: &[i16], + rep_levels: &[i16], + max_def_level: i16, + max_rep_level: i16, +) -> Box { + use crate::arrow::array_reader::PrimitiveArrayReader; + use crate::arrow::arrow_reader::DEFAULT_BATCH_SIZE; + use crate::util::InMemoryPageIterator; + use crate::util::test_common::page_util::{DataPageBuilder, DataPageBuilderImpl}; + + let leaf_type = Type::primitive_type_builder("leaf", PhysicalType::INT32) + .build() + .unwrap(); - fn skip_records(&mut self, num_records: usize) -> Result { - let array = self.next_batch(num_records)?; - Ok(array.len()) - } + let desc = Arc::new(ColumnDescriptor::new( + Arc::new(leaf_type), + max_def_level, + max_rep_level, + ColumnPath::new(vec![]), + )); - fn get_def_levels(&self) -> Option<&[i16]> { - self.def_levels - .as_ref() - .map(|l| &l[self.last_idx..self.cur_idx]) + let mut pb = DataPageBuilderImpl::new(desc.clone(), def_levels.len() as u32, true); + if max_rep_level > 0 { + pb.add_rep_levels(max_rep_level, rep_levels); } - - fn get_rep_levels(&self) -> Option<&[i16]> { - self.rep_levels - .as_ref() - .map(|l| &l[self.last_idx..self.cur_idx]) + if max_def_level > 0 { + pb.add_def_levels(max_def_level, def_levels); } + pb.add_values::(Encoding::PLAIN, values); + + let pages = vec![vec![pb.consume()]]; + let page_iter = InMemoryPageIterator::new(pages); + Box::new( + PrimitiveArrayReader::::new(Box::new(page_iter), desc, None, DEFAULT_BATCH_SIZE) + .unwrap(), + ) } /// Iterator for testing reading empty columns diff --git a/parquet/src/arrow/arrow_reader/read_plan.rs b/parquet/src/arrow/arrow_reader/read_plan.rs index ac2e105ecf4f..674ae2b8d964 100644 --- a/parquet/src/arrow/arrow_reader/read_plan.rs +++ b/parquet/src/arrow/arrow_reader/read_plan.rs @@ -536,11 +536,9 @@ mod tests { fn with_predicate_options_limit_pads_tail_when_no_prior_selection() { use crate::arrow::ProjectionMask; use crate::arrow::array_reader::StructArrayReader; - use crate::arrow::array_reader::test_util::InMemoryArrayReader; + use crate::arrow::array_reader::test_util::make_int32_page_reader; use crate::arrow::arrow_reader::ArrowPredicateFn; - use arrow_array::Int32Array; use arrow_schema::{DataType as ArrowType, Field, Fields}; - use std::sync::Arc; // 100 rows, all match the predicate. Limit stops the loop after 10 // matches — but the resulting RowSelection must still describe the @@ -550,14 +548,14 @@ mod tests { const LIMIT: usize = 10; let data: Vec = (0..TOTAL_ROWS as i32).collect(); - let array = Arc::new(Int32Array::from(data)); - let leaf = InMemoryArrayReader::new(ArrowType::Int32, array.clone(), None, None); + let levels = vec![0; TOTAL_ROWS]; + let leaf = make_int32_page_reader(&data, &levels, &levels, 0, 0); let struct_type = ArrowType::Struct(Fields::from(vec![Field::new( "c0", ArrowType::Int32, false, )])); - let struct_reader = StructArrayReader::new(struct_type, vec![Box::new(leaf)], 0, 0, false); + let struct_reader = StructArrayReader::new(struct_type, vec![leaf], 0, 0, false); let mut predicate = ArrowPredicateFn::new(ProjectionMask::all(), |batch| { Ok(BooleanArray::from(vec![true; batch.num_rows()])) From e6c94488158df687a57e134266323d432f4eba28 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 5 May 2026 13:19:26 -0400 Subject: [PATCH 08/33] Update release schedule on README (#9881) # Which issue does this PR close? - relate to https://github.com/apache/arrow-rs/issues/7392 # Rationale for this change I need to keep this stuff written down somewhere otherwise I lose track of it # What changes are included in this PR? Update the release schedule of the README to reflect - https://github.com/apache/arrow-rs/issues/7392 # Are these changes tested? # Are there any user-facing changes? --- README.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0735dae57cdc..7ea4205d56f5 100644 --- a/README.md +++ b/README.md @@ -91,13 +91,23 @@ Planned Release Schedule | Approximate Date | Version | Notes | | ---------------- | ---------- | --------------------------------------- | -| March 2026 | [`58.1.0`] | Minor, NO breaking API changes | -| April 2026 | [`58.2.0`] | Minor, NO breaking API changes | +| May 2026 | [`58.3.0`] | Minor, NO breaking API changes | +| May 2026 | [`57.3.1`] | Patch, NO breaking API changes | +| May 2026 | [`56.2.1`] | Patch, NO breaking API changes | | May 2026 | [`59.0.0`] | Major, potentially breaking API changes | +| June 2026 | [`59.1.0`] | Minor, NO breaking API changes | +| July 2026 | [`59.2.0`] | Minor, NO breaking API changes | +| August 2026 | [`60.0.0`] | Major, potentially breaking API changes | [`58.1.0`]: https://github.com/apache/arrow-rs/issues/9108 [`58.2.0`]: https://github.com/apache/arrow-rs/issues/9109 +[`58.3.0`]: https://github.com/apache/arrow-rs/issues/9859 +[`57.3.1`]: https://github.com/apache/arrow-rs/issues/9858 +[`56.2.1`]: https://github.com/apache/arrow-rs/issues/9857 [`59.0.0`]: https://github.com/apache/arrow-rs/issues/9110 +[`59.1.0`]: https://github.com/apache/arrow-rs/issues/9878 +[`59.2.0`]: https://github.com/apache/arrow-rs/issues/9879 +[`60.0.0`]: https://github.com/apache/arrow-rs/issues/9880 [ticket #5368]: https://github.com/apache/arrow-rs/issues/5368 [semantic versioning]: https://semver.org/ From 3790d88b5e03a0b68c68a288770c980b63772ccb Mon Sep 17 00:00:00 2001 From: Sebastian Rabenhorst <4246554+rabenhorst@users.noreply.github.com> Date: Tue, 5 May 2026 20:22:32 +0200 Subject: [PATCH 09/33] Pre-size dedup HashTable in GenericByteDictionaryBuilder::with_capacity (#9908) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Which issue does this PR close? - Closes #9907. # Rationale for this change `GenericByteDictionaryBuilder::with_capacity` left the internal `dedup` `HashTable` at default capacity (0), forcing 0 → 4 → 8 → 16 → 32 → 64 → … resize+rehash cycles on the first inserts even when callers passed a non-zero `value_capacity`. `new()` already sizes `dedup` from `keys_builder.capacity()`; `with_capacity` (the variant intended to amortize allocations) should do the same from `value_capacity`. As-is, callers reaching for `with_capacity` to avoid alloc churn get the opposite of what they asked for on the dedup path. See #9907 for the full context including a CPU profile showing `GenericByteDictionaryBuilder::append` at 7.24% cum where ~half is the resize+rehash cost on the first batch. # What changes are included in this PR? - `arrow-array/src/builder/generic_bytes_dictionary_builder.rs`: change `dedup: Default::default()` to `dedup: HashTable::with_capacity(value_capacity)` in `with_capacity`. One-line behavioral change. - Regression test `test_with_capacity_presizes_dedup` asserting `dedup.capacity() >= value_capacity` after construction. This fixes `StringDictionaryBuilder`, `BinaryDictionaryBuilder`, and the `Large*` variants (all type aliases over `GenericByteDictionaryBuilder`). # Are these changes tested? Yes. New test in `arrow-array/src/builder/generic_bytes_dictionary_builder.rs::tests`: ```rust #[test] fn test_with_capacity_presizes_dedup() { // `with_capacity` must size the dedup `HashTable` from `value_capacity`, // otherwise the first inserts force a chain of resize+rehash cycles. let value_capacity = 128; let builder = GenericByteDictionaryBuilder::>::with_capacity( 256, value_capacity, value_capacity * 32, ); assert!( builder.dedup.capacity() >= value_capacity, "dedup HashTable not pre-sized: got capacity {}, expected >= {}", builder.dedup.capacity(), value_capacity, ); } ``` Verified the test fails on unpatched `main` (`got capacity 0, expected >= 128`) and passes with the fix. Full `generic_bytes_dictionary_builder` test suite (17 tests) still passes. `cargo clippy --lib -- -D warnings` is clean. I did not add a benchmark — the asymptotic behavior is what matters here (eliminating O(value_capacity) resize+rehash work on first use), and the existing arrow-rs benchmark suite already exercises dictionary builders. Happy to add one if reviewers prefer. # Are there any user-facing changes? No public API changes. Behavior change is strictly an allocation/perf improvement: callers of `with_capacity` who previously got a zero-capacity dedup table now get one sized to `value_capacity`. This means a slightly larger upfront allocation in exchange for avoiding the resize+rehash chain. Callers passing very large `value_capacity` values intentionally to hint the values buffer (but not expecting to insert that many distinct values) will now also pre-allocate that much dedup capacity — this matches the documented contract of the parameter (`"the number of distinct dictionary values"`). --- ## AI-assisted submission disclosure Per `CONTRIBUTING.md` — this PR was prepared with AI assistance (Claude). The bug was found while profiling a downstream Arrow producer; the fix and regression test were reviewed and verified locally. The change is one line plus a focused test, and I am happy to debug and own follow-ups. Co-authored-by: AI --- .../generic_bytes_dictionary_builder.rs | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/arrow-array/src/builder/generic_bytes_dictionary_builder.rs b/arrow-array/src/builder/generic_bytes_dictionary_builder.rs index 956014a72b7d..4ce451aec81e 100644 --- a/arrow-array/src/builder/generic_bytes_dictionary_builder.rs +++ b/arrow-array/src/builder/generic_bytes_dictionary_builder.rs @@ -84,7 +84,7 @@ where ) -> Self { Self { state: Default::default(), - dedup: Default::default(), + dedup: HashTable::with_capacity(value_capacity), keys_builder: PrimitiveBuilder::with_capacity(keys_capacity), values_builder: GenericByteBuilder::::with_capacity(value_capacity, data_capacity), } @@ -647,6 +647,25 @@ mod tests { test_bytes_dictionary_builder::>(vec![b"abc", b"def"]); } + #[test] + fn test_with_capacity_presizes_dedup() { + // `with_capacity` must size the dedup `HashTable` from `value_capacity`, + // otherwise the first inserts force a chain of resize+rehash cycles. + let value_capacity = 128; + let builder = + GenericByteDictionaryBuilder::>::with_capacity( + 256, + value_capacity, + value_capacity * 32, + ); + assert!( + builder.dedup.capacity() >= value_capacity, + "dedup HashTable not pre-sized: got capacity {}, expected >= {}", + builder.dedup.capacity(), + value_capacity, + ); + } + fn test_bytes_dictionary_builder_finish_cloned(values: Vec<&T::Native>) where T: ByteArrayType, From 1d89737b4dc13a603f7e88a7ca20d571120b9b5f Mon Sep 17 00:00:00 2001 From: masumi-ryugo Date: Wed, 6 May 2026 03:33:18 +0900 Subject: [PATCH 10/33] fix(parquet): Prevent negative list sizes in Thrift compact protocol parser (#9868) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Which issue does this PR close? Closes (in part) #9874. That issue tracks both this immediate fix and the broader thrift-parser hardening @etseidl mentioned in review. Originally found via `cargo-fuzz` libFuzzer harness over `ParquetMetaDataReader::parse_and_finish` and `ParquetRecordBatchReaderBuilder::try_new` — ~95 unique crashing inputs all converged on this single root cause. # Rationale for this change `parquet::parquet_thrift::read_thrift_vec` reads a Thrift compact-protocol list header and then calls `Vec::with_capacity(list_ident.size as usize)` where `list_ident.size` is a 32-bit varint pulled directly from attacker-controlled bytes. On a malformed input the value can be close to `i32::MAX`, which after the per-element-size multiplication is a multi-GB-to-multi-hundred-GB allocation request. That panics in `alloc::raw_vec::handle_error` with `capacity overflow` (or OOM-kills the process on smaller-but-still-huge values) before any element is decoded. Every public sync API that parses parquet bytes funnels through this function: - `ParquetMetaDataReader::parse_and_finish` - `ParquetMetaDataReader::decode_metadata` - `SerializedFileReader::new` - `ParquetRecordBatchReaderBuilder::try_new` - the `async_reader` family (re-exports `decode_metadata` after prefetching footer bytes) So any downstream code that hands attacker-controlled bytes to a parquet reader gets a panic-on-decode DoS. Per `SECURITY.md` this is a bug, not a vulnerability — there is no information disclosure or RCE path, only availability — but it is reachable from every metadata entry point and worth closing. # What changes are included in this PR? - Add a default `remaining_bytes() -> Option` method to `ThriftCompactInputProtocol`, returning the number of bytes still available when the protocol is backed by an in-memory buffer. - Override it on `ThriftSliceInputProtocol` to return `Some(self.buf.len())`. - In `read_thrift_vec`, clamp the capacity passed to `Vec::with_capacity` by `remaining_bytes()` (every Thrift element costs at least one wire byte, so a list whose declared `size` exceeds the remaining input is malformed by definition — the same clamp the C++ Thrift runtime applies). Streaming protocols that cannot report a remaining count fall back to a 1 MiB cap and grow on push. - Reject negative sizes explicitly. Although the wire format is unsigned, a hostile encoder can construct a varint that decodes to a negative `i32`. - Three regression tests in the existing `parquet_thrift::tests` module: - `test_read_thrift_vec_huge_size_does_not_panic` exercises the direct `read_thrift_vec` path with a varint that decodes to ~`i32::MAX`. - `test_decode_metadata_huge_thrift_list_does_not_panic` runs the 10-byte fuzzer repro through `ParquetMetaDataReader::decode_metadata` and asserts a clean `Err`. - `test_read_thrift_vec_negative_size_returns_err` asserts the explicit negative-size guard. # Are these changes tested? Yes — three new unit tests as above. `cargo test -p parquet --release`, `cargo clippy -p parquet --all-targets -- -D warnings`, and `cargo fmt --check` are all clean. (Note: a single pre-existing `should_panic` test `test_read_non_utf8_binary_as_utf8` aborts during the rust test harness's panic-formatting machinery on `origin/main` HEAD itself, so it is excluded with `--skip` for the local run. Unrelated to this PR.) # Are there any user-facing changes? Yes — malformed Parquet inputs that previously triggered a `capacity overflow` panic during metadata decoding now return a `ParquetError` that callers can handle. No behavior change for well-formed files. # Reproducer 10 bytes: ``` 0x28 0xfc 0xfc 0xfc 0xfc 0xfc 0xfc 0xfc 0xfc 0x51 ``` ```rust let bytes: &[u8] = &[0x28, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0x51]; let _ = parquet::file::metadata::ParquetMetaDataReader::decode_metadata(bytes); ``` Before this PR (RUST_BACKTRACE=1): ``` panicked at library/alloc/src/raw_vec/mod.rs:28:5: capacity overflow 3: alloc::raw_vec::handle_error 4: alloc::raw_vec::with_capacity_in 7: alloc::vec::Vec::with_capacity:: 8: parquet::parquet_thrift::read_thrift_vec (parquet_thrift.rs:690) 9: parquet::file::metadata::thrift::parquet_metadata_from_bytes (thrift/mod.rs:790) 10: parquet::file::metadata::parser::decode_metadata (parser.rs:233) 13: ParquetMetaDataReader::decode_footer_metadata (reader.rs:783) 14: ParquetMetaDataReader::parse_metadata (reader.rs:585) 17: ParquetMetaDataReader::parse_and_finish (reader.rs:230) ``` After this PR: `Err(ParquetError::EOF("Unexpected EOF"))` (or similar shape depending on which struct field fails first), no panic. A 45-byte reproducer that drives the full reader stack (with valid `PAR1` magic, so it works for `ParquetRecordBatchReaderBuilder::try_new`) behaves the same way. # Found via `cargo-fuzz` libFuzzer harness wrapping `ParquetMetaDataReader::parse_and_finish` and `ParquetRecordBatchReaderBuilder::try_new` over `bytes::Bytes`. ~95 unique crashing inputs (different VLQ sizes, different element types — `SchemaElement`, `RowGroup`, `ColumnChunk` — different offsets in the metadata graph) all converged on this single root cause within ~3 minutes of single-thread fuzzing. One bug, many surface symptoms. --------- Co-authored-by: masumi ryugo <280057467+masumi-ryugo@users.noreply.github.com> Co-authored-by: Ed Seidl --- parquet/src/parquet_thrift.rs | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/parquet/src/parquet_thrift.rs b/parquet/src/parquet_thrift.rs index 6c82a0bf2c07..a13baa0924e0 100644 --- a/parquet/src/parquet_thrift.rs +++ b/parquet/src/parquet_thrift.rs @@ -36,6 +36,7 @@ use crate::{ write_thrift_field, }; use std::io::Error; +use std::num::TryFromIntError; use std::str::Utf8Error; #[derive(Debug)] @@ -46,6 +47,7 @@ pub(crate) enum ThriftProtocolError { InvalidElementType(u8), FieldDeltaOverflow { field_delta: u8, last_field_id: i16 }, InvalidBoolean(u8), + IntegerOverflow, Utf8Error, SkipDepth(FieldType), SkipUnsupportedType(FieldType), @@ -70,6 +72,9 @@ impl From for ParquetError { ThriftProtocolError::InvalidBoolean(value) => { general_err!("cannot convert {} into bool", value) } + ThriftProtocolError::IntegerOverflow => { + general_err!("integer overflow decoding thrift value") + } ThriftProtocolError::Utf8Error => general_err!("invalid utf8"), ThriftProtocolError::SkipDepth(field_type) => { general_err!("cannot parse past {:?}", field_type) @@ -94,6 +99,13 @@ impl From for ThriftProtocolError { } } +impl From for ThriftProtocolError { + fn from(_: TryFromIntError) -> Self { + // ignore error payload to reduce the size of ThriftProtocolError + Self::IntegerOverflow + } +} + pub type ThriftProtocolResult = Result; /// Wrapper for thrift `double` fields. This is used to provide @@ -317,7 +329,13 @@ pub(crate) trait ThriftCompactInputProtocol<'a> { // high bits set high if count and type encoded separately possible_element_count as i32 } else { - self.read_vlq()? as _ + // The list size on the wire is an unsigned varint, but we represent + // it as `i32` (matching Java's `int` and the Thrift schema). + // A varint that decodes above `i32::MAX` is malformed input — reject + // it here at the protocol layer rather than letting the cast wrap + // into a negative size that downstream allocation code has to + // re-validate. + i32::try_from(self.read_vlq()?)? }; Ok(ListIdentifier { @@ -1106,4 +1124,18 @@ pub(crate) mod tests { assert_eq!(header.size, 0); assert_eq!(header.element_type, ElementType::Byte); } + + /// A Thrift list header whose `size` varint decodes above `i32::MAX` + /// must be rejected at the protocol layer rather than wrapping into a + /// negative `i32` and being smuggled into downstream allocation code. + #[test] + fn test_read_list_begin_size_above_i32_max_returns_err() { + // List header: element_type=8 (Binary), 0xF=follow-up varint. + // Varint 80 80 80 80 08 decodes to 0x8000_0000 = i32::MAX + 1. + let mut data: Vec = vec![0xF8]; + data.extend_from_slice(&[0x80, 0x80, 0x80, 0x80, 0x08]); + let mut prot = ThriftSliceInputProtocol::new(&data); + let result = prot.read_list_begin(); + assert!(result.is_err(), "expected error, got {result:?}"); + } } From 8091f3f17b2de355f7c47e7a0907000d308f8f3e Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 5 May 2026 15:08:06 -0400 Subject: [PATCH 11/33] [arrow-array] Use consistent `value_length` name in FixedSizeBinaryArray (#9905) # Which issue does this PR close? - Part of https://github.com/apache/arrow-rs/issues/9906 - First follow on to https://github.com/apache/arrow-rs/pull/9872 # Rationale for this change While trying to avoid overflows due to using i32 arithmetic in FixedSizeBinaryArray, I found the use of the term `size` in parameters to be confusing when the field name is called `value_length` # What changes are included in this PR? Change several parameter / variable names to `value_length` to keep the code consistent # Are these changes tested? By CI # Are there any user-facing changes? No this is an internal code refactor --- .../src/array/fixed_size_binary_array.rs | 150 +++++++++--------- 1 file changed, 79 insertions(+), 71 deletions(-) diff --git a/arrow-array/src/array/fixed_size_binary_array.rs b/arrow-array/src/array/fixed_size_binary_array.rs index 3c3dc8f58920..a54496addc5d 100644 --- a/arrow-array/src/array/fixed_size_binary_array.rs +++ b/arrow-array/src/array/fixed_size_binary_array.rs @@ -96,44 +96,48 @@ pub struct FixedSizeBinaryArray { } impl FixedSizeBinaryArray { - /// Create a new [`FixedSizeBinaryArray`] with `size` element size, panicking on failure + /// Create a new [`FixedSizeBinaryArray`] with `value_length` bytes per element, panicking on + /// failure /// /// # Panics /// /// Panics if [`Self::try_new`] returns an error - pub fn new(size: i32, values: Buffer, nulls: Option) -> Self { - Self::try_new(size, values, nulls).unwrap() + pub fn new(value_length: i32, values: Buffer, nulls: Option) -> Self { + Self::try_new(value_length, values, nulls).unwrap() } /// Create a new [`Scalar`] from `value` pub fn new_scalar(value: impl AsRef<[u8]>) -> Scalar { let v = value.as_ref(); - let size = i32::try_from(v.len()).expect("FixedSizeBinaryArray value length exceeds i32"); - Scalar::new(Self::new(size, Buffer::from(v), None)) + let value_length = + i32::try_from(v.len()).expect("FixedSizeBinaryArray value length exceeds i32"); + Scalar::new(Self::new(value_length, Buffer::from(v), None)) } /// Create a new [`FixedSizeBinaryArray`] from the provided parts, returning an error on failure /// - /// Creating an arrow with `size == 0` will try to get the length from the null buffer. If - /// no null buffer is provided, the resulting array will have length zero. + /// Creating an array with `value_length == 0` will try to get the length from the null + /// buffer. If no null buffer is provided, the resulting array will have length zero. /// /// # Errors /// - /// * `size < 0` - /// * `values.len() / size != nulls.len()` - /// * `size == 0 && values.len() != 0` - /// * `len * size > i32::MAX` + /// * `value_length < 0` + /// * `values.len() / value_length != nulls.len()` + /// * `value_length == 0 && values.len() != 0` + /// * `len * value_length > i32::MAX` pub fn try_new( - size: i32, + value_length: i32, values: Buffer, nulls: Option, ) -> Result { - let data_type = DataType::FixedSizeBinary(size); - let s = size.to_usize().ok_or_else(|| { - ArrowError::InvalidArgumentError(format!("Size cannot be negative, got {size}")) + let data_type = DataType::FixedSizeBinary(value_length); + let value_size = value_length.to_usize().ok_or_else(|| { + ArrowError::InvalidArgumentError(format!( + "Value length cannot be negative, got {value_length}" + )) })?; - let len = match values.len().checked_div(s) { + let len = match values.len().checked_div(value_size) { Some(len) => { if let Some(n) = nulls.as_ref() { if n.len() != len { @@ -150,21 +154,21 @@ impl FixedSizeBinaryArray { None => { if !values.is_empty() { return Err(ArrowError::InvalidArgumentError( - "Buffer cannot have non-zero length if the item size is zero".to_owned(), + "Buffer cannot have non-zero length if the value length is zero".to_owned(), )); } - // If the item size is zero, try to determine the length from the null buffer + // If the value length is zero, try to determine the length from the null buffer nulls.as_ref().map(|n| n.len()).unwrap_or(0) } }; - Self::validate_lengths(s, len)?; + Self::validate_lengths(value_size, len)?; Ok(Self { data_type, value_data: values, - value_length: size, + value_length, nulls, len, }) @@ -203,21 +207,21 @@ impl FixedSizeBinaryArray { /// /// Panics if /// - /// * `size < 0` - /// * `size * len` would overflow `usize` - /// * `size * len > i32::MAX` - /// * `size * len * 8` would overflow `usize` - pub fn new_null(size: i32, len: usize) -> Self { + /// * `value_length < 0` + /// * `value_length * len` would overflow `usize` + /// * `value_length * len > i32::MAX` + /// * `value_length * len * 8` would overflow `usize` + pub fn new_null(value_length: i32, len: usize) -> Self { const BITS_IN_A_BYTE: usize = 8; - let size_usize = size.to_usize().unwrap(); - Self::validate_lengths(size_usize, len).unwrap(); - let capacity_in_bytes = size_usize.checked_mul(len).unwrap(); + let value_size = value_length.to_usize().unwrap(); + Self::validate_lengths(value_size, len).unwrap(); + let capacity_in_bytes = value_size.checked_mul(len).unwrap(); let capacity_in_bits = capacity_in_bytes.checked_mul(BITS_IN_A_BYTE).unwrap(); Self { - data_type: DataType::FixedSizeBinary(size), + data_type: DataType::FixedSizeBinary(value_length), value_data: MutableBuffer::new_null(capacity_in_bits).into(), nulls: Some(NullBuffer::new_null(len)), - value_length: size, + value_length, len, } } @@ -351,7 +355,7 @@ impl FixedSizeBinaryArray { U: AsRef<[u8]>, { let mut len = 0; - let mut size = None; + let mut value_size = None; let mut byte = 0; let iter_size_hint = iter.size_hint().0; @@ -369,7 +373,7 @@ impl FixedSizeBinaryArray { if let Some(slice) = item { let slice = slice.as_ref(); - if let Some(size) = size { + if let Some(size) = value_size { if size != slice.len() { return Err(ArrowError::InvalidArgumentError(format!( "Nested array size mismatch: one is {}, and the other is {}", @@ -379,7 +383,7 @@ impl FixedSizeBinaryArray { } } else { let len = slice.len(); - size = Some(len); + value_size = Some(len); // Now that we know how large each element is we can reserve // sufficient capacity in the underlying mutable buffer for // the data. @@ -396,7 +400,7 @@ impl FixedSizeBinaryArray { } bit_util::set_bit(null_buf.as_slice_mut(), len); buffer.extend_from_slice(slice); - } else if let Some(size) = size { + } else if let Some(size) = value_size { buffer.extend_zeros(size); } else { prepend += 1; @@ -415,18 +419,18 @@ impl FixedSizeBinaryArray { let nulls = NullBuffer::from_unsliced_buffer(null_buf, len); - let size = size.unwrap_or(0); - Self::validate_lengths(size, len)?; - let size = size.try_into().map_err(|_| { + let value_size = value_size.unwrap_or(0); + Self::validate_lengths(value_size, len)?; + let value_length = value_size.try_into().map_err(|_| { ArrowError::InvalidArgumentError(format!( - "FixedSizeBinaryArray value length exceeds i32, got {size}" + "FixedSizeBinaryArray value length exceeds i32, got {value_size}" )) })?; Ok(Self { - data_type: DataType::FixedSizeBinary(size), + data_type: DataType::FixedSizeBinary(value_length), value_data: buffer.into(), nulls, - value_length: size, + value_length, len, }) } @@ -434,8 +438,8 @@ impl FixedSizeBinaryArray { /// Create an array from an iterable argument of sparse byte slices. /// Sparsity means that items returned by the iterator are optional, i.e input argument can /// contain `None` items. In cases where the iterator returns only `None` values, this - /// also takes a size parameter to ensure that the a valid FixedSizeBinaryArray is still - /// created. + /// also takes a `value_length` parameter to ensure that a valid + /// [`FixedSizeBinaryArray`] is still created. /// /// # Examples /// @@ -455,22 +459,27 @@ impl FixedSizeBinaryArray { /// # Errors /// /// Returns error if argument has length zero, or sizes of nested slices don't match. - pub fn try_from_sparse_iter_with_size(mut iter: T, size: i32) -> Result + pub fn try_from_sparse_iter_with_size( + mut iter: T, + value_length: i32, + ) -> Result where T: Iterator>, U: AsRef<[u8]>, { - let size_usize = size.to_usize().ok_or_else(|| { - ArrowError::InvalidArgumentError(format!("Size cannot be negative, got {size}")) + let value_size = value_length.to_usize().ok_or_else(|| { + ArrowError::InvalidArgumentError(format!( + "Value length cannot be negative, got {value_length}" + )) })?; let mut len = 0; let mut byte = 0; let iter_size_hint = iter.size_hint().0; let mut null_buf = MutableBuffer::new(bit_util::ceil(iter_size_hint, 8)); - let capacity = iter_size_hint.checked_mul(size_usize).ok_or_else(|| { + let capacity = iter_size_hint.checked_mul(value_size).ok_or_else(|| { ArrowError::InvalidArgumentError(format!( - "FixedSizeBinaryArray error: value size {size_usize} * len hint {iter_size_hint} exceeds usize" + "FixedSizeBinaryArray error: value size {value_size} * len hint {iter_size_hint} exceeds usize" )) })?; let mut buffer = MutableBuffer::new(capacity); @@ -485,10 +494,10 @@ impl FixedSizeBinaryArray { if let Some(slice) = item { let slice = slice.as_ref(); - if size_usize != slice.len() { + if value_size != slice.len() { return Err(ArrowError::InvalidArgumentError(format!( "Nested array size mismatch: one is {}, and the other is {}", - size, + value_length, slice.len() ))); } @@ -496,7 +505,7 @@ impl FixedSizeBinaryArray { bit_util::set_bit(null_buf.as_slice_mut(), len); buffer.extend_from_slice(slice); } else { - buffer.extend_zeros(size_usize); + buffer.extend_zeros(value_size); } len += 1; @@ -505,14 +514,14 @@ impl FixedSizeBinaryArray { })?; let nulls = NullBuffer::from_unsliced_buffer(null_buf, len); - Self::validate_lengths(size_usize, len)?; + Self::validate_lengths(value_size, len)?; Ok(Self { - data_type: DataType::FixedSizeBinary(size), + data_type: DataType::FixedSizeBinary(value_length), value_data: buffer.into(), nulls, len, - value_length: size, + value_length, }) } @@ -539,23 +548,22 @@ impl FixedSizeBinaryArray { U: AsRef<[u8]>, { let mut len = 0; - let mut size = None; + let mut value_size = None; let iter_size_hint = iter.size_hint().0; let mut buffer = MutableBuffer::new(0); iter.try_for_each(|item| -> Result<(), ArrowError> { let slice = item.as_ref(); - if let Some(size) = size { - if size != slice.len() { + if let Some(value_size) = value_size { + if value_size != slice.len() { return Err(ArrowError::InvalidArgumentError(format!( - "Nested array size mismatch: one is {}, and the other is {}", - size, + "Nested array size mismatch: one is {value_size}, and the other is {}", slice.len() ))); } } else { let len = slice.len(); - size = Some(len); + value_size = Some(len); if let Some(capacity) = iter_size_hint.checked_mul(len) { buffer.reserve(capacity); } @@ -574,18 +582,18 @@ impl FixedSizeBinaryArray { )); } - let size = size.unwrap_or(0); - Self::validate_lengths(size, len)?; - let size = size.try_into().map_err(|_| { + let value_size = value_size.unwrap_or(0); + Self::validate_lengths(value_size, len)?; + let value_length = value_size.try_into().map_err(|_| { ArrowError::InvalidArgumentError(format!( - "FixedSizeBinaryArray value length exceeds i32, got {size}" + "FixedSizeBinaryArray value length exceeds i32, got {value_size}" )) })?; Ok(Self { - data_type: DataType::FixedSizeBinary(size), + data_type: DataType::FixedSizeBinary(value_length), value_data: buffer.into(), nulls: None, - value_length: size, + value_length, len, }) } @@ -615,14 +623,14 @@ impl From for FixedSizeBinaryArray { _ => panic!("Expected data type to be FixedSizeBinary"), }; - let size = value_length + let value_size = value_length .to_usize() .expect("FixedSizeBinaryArray value length must be non-negative"); - Self::validate_lengths(size, len) + Self::validate_lengths(value_size, len) .expect("FixedSizeBinaryArray offsets must fit within i32"); let value_data = buffers[0].slice_with_length( - offset.checked_mul(size).expect("offset overflow"), - len.checked_mul(size).expect("length overflow"), + offset.checked_mul(value_size).expect("offset overflow"), + len.checked_mul(value_size).expect("length overflow"), ); Self { @@ -1157,7 +1165,7 @@ mod tests { assert_eq!( err.to_string(), - "Invalid argument error: Size cannot be negative, got -1" + "Invalid argument error: Value length cannot be negative, got -1" ); let nulls = NullBuffer::new_null(3); @@ -1182,7 +1190,7 @@ mod tests { FixedSizeBinaryArray::try_new(0, buffer, None).unwrap_err(); assert_eq!( zero_sized_with_non_empty_buffer_err.to_string(), - "Invalid argument error: Buffer cannot have non-zero length if the item size is zero" + "Invalid argument error: Buffer cannot have non-zero length if the value length is zero" ); } } From 5bf891265174ee85310582f9d1fa3be844d38fdd Mon Sep 17 00:00:00 2001 From: albertlockett Date: Wed, 6 May 2026 09:33:14 -0400 Subject: [PATCH 12/33] [arrow-string]: add `like::eq_ascii_ignore_case` kernel (#9871) # Which issue does this PR close? - Closes https://github.com/apache/arrow-rs/issues/9870 # Rationale for this change Users might wish to have a mechanism to do check if two strings are equal in a case-insensitive way. We already have some machinery to do this in the arrow-string crate (in the `predicate` module). This PR exposes a function so it can be invoked easily. # What changes are included in this PR? Adds a function `like::eq_ascii_ignore_case` to the arrow-string crate that performs a case-insensitive equals check on two string arrays. # Are these changes tested? yes unit tests # Are there any user-facing changes? Yes this function is public --- arrow-string/src/like.rs | 59 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/arrow-string/src/like.rs b/arrow-string/src/like.rs index 55fe292ecb76..2ad7e1cfd961 100644 --- a/arrow-string/src/like.rs +++ b/arrow-string/src/like.rs @@ -15,7 +15,11 @@ // specific language governing permissions and limitations // under the License. -//! Provide SQL's LIKE operators for Arrow's string arrays +//! String predicate kernels for Arrow arrays. +//! +//! Provides SQL `LIKE`/`ILIKE` kernels as well as related +//! string predicates such as `contains`, `starts_with`, `ends_with`, and +//! ASCII case-insensitive equality. use crate::predicate::Predicate; @@ -34,6 +38,7 @@ pub(crate) enum Op { Like(bool), ILike(bool), Contains, + EqIgnoreAsciiCase, StartsWith, EndsWith, } @@ -46,6 +51,7 @@ impl std::fmt::Display for Op { Op::ILike(false) => write!(f, "ILIKE"), Op::ILike(true) => write!(f, "NILIKE"), Op::Contains => write!(f, "CONTAINS"), + Op::EqIgnoreAsciiCase => write!(f, "EQ_IGNORE_ASCII_CASE"), Op::StartsWith => write!(f, "STARTS_WITH"), Op::EndsWith => write!(f, "ENDS_WITH"), } @@ -124,7 +130,7 @@ pub fn nilike(left: &dyn Datum, right: &dyn Datum) -> Result Result Result Result Result { + like_op(Op::EqIgnoreAsciiCase, left, right) +} + fn like_op(op: Op, lhs: &dyn Datum, rhs: &dyn Datum) -> Result { use arrow_schema::DataType::*; let (l, l_s) = lhs.get(); @@ -328,6 +358,7 @@ fn op_scalar<'a, T: StringArrayType<'a>>( Op::Like(neg) => Predicate::like(r)?.evaluate_array(l, neg), Op::ILike(neg) => Predicate::ilike(r, l.is_ascii())?.evaluate_array(l, neg), Op::Contains => Predicate::contains(r).evaluate_array(l, false), + Op::EqIgnoreAsciiCase => Predicate::IEqAscii(r).evaluate_array(l, false), Op::StartsWith => Predicate::StartsWith(r).evaluate_array(l, false), Op::EndsWith => Predicate::EndsWith(r).evaluate_array(l, false), }; @@ -362,6 +393,10 @@ fn op_binary<'a>( Op::Like(neg) => binary_predicate(l, r, neg, Predicate::like), Op::ILike(neg) => binary_predicate(l, r, neg, |s| Predicate::ilike(s, false)), Op::Contains => Ok(l.zip(r).map(|(l, r)| Some(str_contains(l?, r?))).collect()), + Op::EqIgnoreAsciiCase => Ok(l + .zip(r) + .map(|(l, r)| Some(Predicate::IEqAscii(l?).evaluate(r?))) + .collect()), Op::StartsWith => Ok(l .zip(r) .map(|(l, r)| Some(Predicate::StartsWith(r?).evaluate(l?))) @@ -1394,6 +1429,22 @@ mod tests { vec![true, false, true, true, true] ); + test_utf8!( + test_utf8_array_eq_ignore_ascii_case, + vec!["arrow", "arrow", "arrow", "arrow", "parquet", "parquet"], + vec!["arrow", "ARROW", "arro", "aRrOw", "arrow", "ARROW"], + eq_ignore_ascii_case, + vec![true, true, false, true, false, false] + ); + + test_utf8_scalar!( + test_utf8_array_eq_ignore_ascii_case_scalar, + vec!["arrow", "aRrOW", "arro", "ARROW", "parquet", "PARQUET"], + "arrow", + eq_ignore_ascii_case, + vec![true, true, false, true, false, false] + ); + #[test] fn test_dict_like_kernels() { let data = vec![ From 7f6524def267f5c5be73b7d5320185ea9f3bb91f Mon Sep 17 00:00:00 2001 From: masumi-ryugo Date: Thu, 7 May 2026 00:02:47 +0900 Subject: [PATCH 13/33] fix(parquet): bound schema num_children before Vec::with_capacity (#9884) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Validate `SchemaElement::num_children` in `parquet::schema::types::schema_from_array_helper` before passing it to `Vec::with_capacity`. Reject: 1. **Negative counts** — they wrap to a huge `usize` on the cast, and they're nonsensical anyway (`num_children` is a count). 2. **Counts that exceed the remaining schema-array entries** — each declared child must consume at least one entry, so a value larger than `num_elements - index - 1` is invalid input. Without this bound the cast feeds `Vec::with_capacity(~i32::MAX)` and `alloc::raw_vec::handle_error` panics with `capacity overflow` on a 64-bit target. ```rust if n < 0 { return Err(general_err!(...)); } let n_children = n as usize; let remaining = num_elements.saturating_sub(index + 1); if n_children > remaining { return Err(general_err!(...)); } let mut fields = Vec::with_capacity(n_children); ``` ## How it was found The `parquet/fuzz/thrift_decode` cargo-fuzz harness from #5332 / [`fuzz/initial-harnesses`](https://github.com/masumi-ryugo/arrow-rs/tree/fuzz/initial-harnesses), running for ~30 s on `parquet/main` HEAD with no seed corpus, produces this 22-byte input: ``` 00000000 02 00 2b 11 01 08 00 11 11 00 00 00 00 00 00 00 |..+.............| 00000010 f7 f9 11 01 a5 ff |......| ``` Backtrace before this patch: ``` thread '' panicked at raw_vec/mod.rs:28:5: capacity overflow 2: alloc::raw_vec::capacity_overflow 3: alloc::raw_vec::handle_error raw_vec/mod.rs:889:29 4: schema_from_array_helper schema/types.rs:1404 5: parquet_schema_from_array schema/types.rs:1304:17 6: parquet_metadata_from_bytes metadata/thrift/mod.rs:791:31 7: decode_metadata metadata/parser.rs:233:5 8: decode_metadata metadata/reader.rs:823:9 ``` After this patch the same bytes return a clean `ParquetError::General`, no panic, no allocation spike. ## Tests Three new tests in `schema::types::tests`: - `test_parquet_schema_from_array_rejects_negative_num_children` — direct `parquet_schema_from_array` call with `num_children = Some(-1)`. - `test_parquet_schema_from_array_rejects_overflowing_num_children` — same with `num_children = Some(i32::MAX)`. - `test_decode_metadata_oom_repro_schema_overflow_does_not_panic` — end-to-end via `ParquetMetaDataReader::decode_metadata` using the 22-byte repro. The existing 107 tests under `schema::` continue to pass. ## Relationship to #9883 Sibling fix, not dependent. #9883 caps the up-front allocation **inside** the thrift parser (`read_thrift_vec`); this PR caps the allocation **downstream** of the thrift parser, where `SchemaElement::num_children` becomes a `Vec::with_capacity` argument. They live on different code paths — fuzz finds inputs that hit either one independently — and either can land first. xref #5332 #9883 #9868 #9874 --------- Co-authored-by: masumi ryugo <280057467+masumi-ryugo@users.noreply.github.com> --- parquet/src/schema/types.rs | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/parquet/src/schema/types.rs b/parquet/src/schema/types.rs index 2c63da74df47..0d504e16fc28 100644 --- a/parquet/src/schema/types.rs +++ b/parquet/src/schema/types.rs @@ -1401,7 +1401,7 @@ fn schema_from_array_helper<'a>( Some(n) => { let repetition = element.repetition_type; - let mut fields = Vec::with_capacity(n as usize); + let mut fields = Vec::with_capacity(usize::try_from(n)?); let mut next_index = index + 1; for _ in 0..n { let child_result = schema_from_array_helper(elements, num_elements, next_index)?; @@ -2517,4 +2517,22 @@ mod tests { let result_schema = parquet_schema_from_array(thrift_schema).unwrap(); assert_eq!(result_schema, expected_schema); } + + #[test] + fn test_parquet_schema_from_array_rejects_negative_num_children() { + let elements = vec![SchemaElement { + r#type: None, + type_length: None, + repetition_type: Some(Repetition::REQUIRED), + name: "schema", + num_children: Some(-1), + converted_type: None, + scale: None, + precision: None, + field_id: None, + logical_type: None, + }]; + let result = parquet_schema_from_array(elements); + assert!(result.unwrap_err().to_string().contains("Integer overflow")); + } } From ded985c95e6d132567710319d21e1901973ea16f Mon Sep 17 00:00:00 2001 From: masumi-ryugo Date: Thu, 7 May 2026 00:06:53 +0900 Subject: [PATCH 14/33] fix(arrow-csv): bound RecordDecoder::flush offset accumulation (#9886) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #9885. ## What `RecordDecoder::flush` walks the per-row offsets emitted by `csv_core::Reader` and accumulates them so each end offset is absolute over `self.data` after the loop. The accumulator was a plain `usize` and the loop body did `*x += offset`, which on malformed input that drives `csv_core` to emit row-relative offsets large enough to wrap a `usize`: - panics with `attempt to add with overflow` in debug builds (and the cargo-fuzz `csv_reader` harness that found this is built with `--debug-assertions`); - silently wraps to a wildly out-of-bounds index in release builds, which then trips an unrelated `assert!` / `unwrap` somewhere downstream. ## Fix Switch the accumulator to `checked_add` and surface the overflow as `ArrowError::CsvError` instead. The body of the loop becomes a normal `for` loop because `?` doesn't compose with the previous closure form. ```rust let mut row_offset: usize = 0; for row in self.offsets[1..self.offsets_len].chunks_exact_mut(self.num_columns) { let offset = row_offset; for x in row.iter_mut() { *x = x.checked_add(offset).ok_or_else(|| { ArrowError::CsvError( "CSV record offsets overflowed usize while flushing".to_string(), ) })?; row_offset = *x; } } ``` ## Repro The cargo-fuzz `csv_reader` harness from [`fuzz/initial-harnesses`](https://github.com/masumi-ryugo/arrow-rs/tree/fuzz/initial-harnesses) (per #5332) reproduces this from an empty corpus in single-digit minutes. The minimized repro is 72 bytes: ``` 0000 2e 22 3f 0a 31 0a 3f 3f 0a 3c 50 50 0a 3f 0a 31 |."?.1.??.' panicked at arrow-csv/src/reader/records.rs:207:21: attempt to add with overflow ``` After this PR the same 72 bytes pass through the fuzz target in 40 ms with exit 0; the API now returns `ArrowError::CsvError(...)` for callers to handle. ## Tests Adds `reader::records::tests::test_flush_offset_overflow_does_not_panic`, which feeds the 72-byte fuzz repro through `RecordDecoder::decode` + `flush` and asserts the loop terminates cleanly instead of panicking. The existing 4 tests in that module continue to pass. ## Alternatives considered - **Cap by `self.data_len`**: each emitted offset is supposed to be ≤ `self.data_len`, so an explicit cap would also turn the overflow into a clean error. I went with `checked_add` because it's the more targeted change — it doesn't add a new invariant on `csv_core`'s output, only refuses to compute something that would have been arithmetically nonsensical anyway. - **Use `saturating_add`**: would silently truncate the offset and then mis-slice `self.data`, producing a confusing `Encountered invalid UTF-8 data` error or panic deeper in the call stack. Worse signal. xref #5332 #9883 #9884 --------- Co-authored-by: masumi ryugo <280057467+masumi-ryugo@users.noreply.github.com> Co-authored-by: Claude Opus 4.7 (1M context) --- arrow-csv/src/reader/records.rs | 36 +++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/arrow-csv/src/reader/records.rs b/arrow-csv/src/reader/records.rs index 33927c93360a..a6d54f867c1d 100644 --- a/arrow-csv/src/reader/records.rs +++ b/arrow-csv/src/reader/records.rs @@ -198,16 +198,21 @@ impl RecordDecoder { // csv_core::Reader writes end offsets relative to the start of the row // Therefore scan through and offset these based on the cumulative row offsets - let mut row_offset = 0; + let mut row_offset: usize = 0; self.offsets[1..self.offsets_len] .chunks_exact_mut(self.num_columns) - .for_each(|row| { + .try_for_each(|row| -> Result<(), ArrowError> { let offset = row_offset; - row.iter_mut().for_each(|x| { - *x += offset; + row.iter_mut().try_for_each(|x| -> Result<(), ArrowError> { + *x = x.checked_add(offset).ok_or_else(|| { + ArrowError::CsvError( + "CSV record offsets overflowed usize while flushing".to_string(), + ) + })?; row_offset = *x; - }); - }); + Ok(()) + }) + })?; // Need to truncate data t1o the actual amount of data read let data = std::str::from_utf8(&self.data[..self.data_len]).map_err(|e| { @@ -399,4 +404,23 @@ mod tests { assert_eq!(read, 5); assert_eq!(bytes, csv.len()); } + + /// Regression test for an overflow path found by the `arrow-csv` + /// cargo-fuzz harness being prototyped for #5332. Stages the + /// `RecordDecoder` state directly so that rebasing the second row's + /// end offset overflows `usize`. With the previous `*x += offset` this + /// panicked with `attempt to add with overflow`; the patched code + /// surfaces the condition as `ArrowError::CsvError`. + #[test] + fn test_flush_offset_overflow_returns_csv_error() { + let mut decoder = RecordDecoder::new(Reader::new(), 1, false); + decoder.offsets = vec![0, usize::MAX, 1]; + decoder.offsets_len = 3; + decoder.num_rows = 2; + let err = decoder.flush().unwrap_err(); + assert_eq!( + err.to_string(), + "Csv error: CSV record offsets overflowed usize while flushing" + ); + } } From 97ff1984910656fcd76be7a2a44b92b032d3b300 Mon Sep 17 00:00:00 2001 From: theirix Date: Thu, 7 May 2026 03:18:46 +0100 Subject: [PATCH 15/33] feat(arrow-string): concat_elements for view, fixed binary (#9876) # Which issue does this PR close? - Closes #9875. # Rationale for this change `concat_elements` module lacks versions for binary view and fixed-size binaries. It's worth having them here. # What changes are included in this PR? - Kernel for `BinaryViewArray` - Kernel for `FixedSizeBinaryArray` - Dispatching logic under `concat_elements_dyn` - Unit tests - # Are these changes tested? New unit tests # Are there any user-facing changes? --- arrow-string/src/concat_elements.rs | 295 +++++++++++++++++++++++++++- 1 file changed, 289 insertions(+), 6 deletions(-) diff --git a/arrow-string/src/concat_elements.rs b/arrow-string/src/concat_elements.rs index 41be8a81cb12..cd4676d28752 100644 --- a/arrow-string/src/concat_elements.rs +++ b/arrow-string/src/concat_elements.rs @@ -18,10 +18,12 @@ //! Provides utility functions for concatenation of elements in arrays. use std::sync::Arc; -use arrow_array::builder::BufferBuilder; +use arrow_array::builder::{ + BinaryViewBuilder, BufferBuilder, FixedSizeBinaryBuilder, StringViewBuilder, +}; use arrow_array::types::ByteArrayType; use arrow_array::*; -use arrow_buffer::{ArrowNativeType, NullBuffer}; +use arrow_buffer::{ArrowNativeType, MutableBuffer, NullBuffer}; use arrow_data::ArrayDataBuilder; use arrow_schema::{ArrowError, DataType}; @@ -168,6 +170,129 @@ pub fn concat_elements_utf8_many( Ok(unsafe { builder.build_unchecked() }.into()) } +/// Returns the elementwise concatenation of a [`FixedSizeBinaryArray`]. +/// +/// The result has `value_length = left.value_length() + right.value_length()`. +/// An index is null if either input is null at that position. +/// +/// An error will be returned if `left` and `right` have different lengths. +pub fn concat_elements_fixed_size_binary( + left: &FixedSizeBinaryArray, + right: &FixedSizeBinaryArray, +) -> Result { + if left.len() != right.len() { + return Err(ArrowError::ComputeError(format!( + "Arrays must have the same length: {} != {}", + left.len(), + right.len() + ))); + } + + let left_size = left.value_length() as usize; + let right_size = right.value_length() as usize; + let output_size = left_size + right_size; + + // Pre-compute combined null bitmap so the per-row NULL check is efficient + let nulls = NullBuffer::union(left.nulls(), right.nulls()); + + let mut result = FixedSizeBinaryBuilder::with_capacity(left.len(), output_size as i32); + let mut buffer = MutableBuffer::with_capacity(output_size); + for i in 0..left.len() { + if nulls.as_ref().is_some_and(|n| n.is_null(i)) { + result.append_null(); + } else { + buffer.clear(); + buffer.extend_from_slice(left.value(i)); + buffer.extend_from_slice(right.value(i)); + result.append_value(&buffer)?; + } + } + + Ok(result.finish()) +} + +/// Concatenates two `BinaryViewArray`s element-wise. +/// If either element is `Null`, the result element is also `Null`. +/// +/// # Errors +/// - Returns an error if the input arrays have different lengths. +/// - Returns an error if any concatenated value exceeds `u32::MAX` in length. +pub fn concat_elements_binary_view_array( + left: &BinaryViewArray, + right: &BinaryViewArray, +) -> Result { + if left.len() != right.len() { + return Err(ArrowError::ComputeError(format!( + "Arrays must have the same length: {} != {}", + left.len(), + right.len() + ))); + } + let mut result = BinaryViewBuilder::with_capacity(left.len()); + + // Avoid reallocations by writing to a reused buffer + let mut buffer = MutableBuffer::new(0); + + // Pre-compute combined null bitmap, so the per-row NULL check is efficient + let nulls = NullBuffer::union(left.nulls(), right.nulls()); + + for i in 0..left.len() { + if nulls.as_ref().is_some_and(|n| n.is_null(i)) { + result.append_null(); + } else { + buffer.clear(); + buffer.extend_from_slice(left.value(i)); + buffer.extend_from_slice(right.value(i)); + result.try_append_value(&buffer)?; + } + } + Ok(result.finish()) +} + +/// Concatenates two `StringViewArray`s element-wise. +/// If either element is `Null`, the result element is also `Null`. +/// +/// # Errors +/// - Returns an error if the input arrays have different lengths. +/// - Returns an error if any concatenated value exceeds `u32::MAX` in length. +/// - Returns an error if concatenated strings do not result in a proper UTF-8 string +// Cannot reuse code with `GenericByteViewBuilder` since `try_append_value` works with +// `AsRef`, and there is no conversion from `ByteViewType` to this or [u8] +pub fn concat_elements_string_view_array( + left: &StringViewArray, + right: &StringViewArray, +) -> Result { + if left.len() != right.len() { + return Err(ArrowError::ComputeError(format!( + "Arrays must have the same length: {} != {}", + left.len(), + right.len() + ))); + } + + let mut result = StringViewBuilder::with_capacity(left.len()); + + // Avoid reallocations by writing to a reused buffer + let mut buffer: Vec = Vec::new(); + + let nulls = NullBuffer::union(left.nulls(), right.nulls()); + + for i in 0..left.len() { + if nulls.as_ref().is_some_and(|n| n.is_null(i)) { + result.append_null(); + } else { + buffer.clear(); + buffer.extend_from_slice(left.value(i).as_bytes()); + buffer.extend_from_slice(right.value(i).as_bytes()); + let s = std::str::from_utf8(&buffer).map_err(|_| { + ArrowError::ComputeError("Concatenated values are not valid UTF-8".into()) + })?; + result.try_append_value(s)?; + } + } + Ok(result.finish()) +} + /// Returns the elementwise concatenation of [`Array`]s. /// /// # Errors @@ -185,22 +310,43 @@ pub fn concat_elements_dyn(left: &dyn Array, right: &dyn Array) -> Result { let left = left.as_any().downcast_ref::().unwrap(); let right = right.as_any().downcast_ref::().unwrap(); - Ok(Arc::new(concat_elements_utf8(left, right).unwrap())) + Ok(Arc::new(concat_elements_utf8(left, right)?)) } (DataType::LargeUtf8, DataType::LargeUtf8) => { let left = left.as_any().downcast_ref::().unwrap(); let right = right.as_any().downcast_ref::().unwrap(); - Ok(Arc::new(concat_elements_utf8(left, right).unwrap())) + Ok(Arc::new(concat_elements_utf8(left, right)?)) } (DataType::Binary, DataType::Binary) => { let left = left.as_any().downcast_ref::().unwrap(); let right = right.as_any().downcast_ref::().unwrap(); - Ok(Arc::new(concat_element_binary(left, right).unwrap())) + Ok(Arc::new(concat_element_binary(left, right)?)) } (DataType::LargeBinary, DataType::LargeBinary) => { let left = left.as_any().downcast_ref::().unwrap(); let right = right.as_any().downcast_ref::().unwrap(); - Ok(Arc::new(concat_element_binary(left, right).unwrap())) + Ok(Arc::new(concat_element_binary(left, right)?)) + } + (DataType::BinaryView, DataType::BinaryView) => { + let left = left.as_any().downcast_ref::().unwrap(); + let right = right.as_any().downcast_ref::().unwrap(); + Ok(Arc::new(concat_elements_binary_view_array(left, right)?)) + } + (DataType::Utf8View, DataType::Utf8View) => { + let left = left.as_any().downcast_ref::().unwrap(); + let right = right.as_any().downcast_ref::().unwrap(); + Ok(Arc::new(concat_elements_string_view_array(left, right)?)) + } + (DataType::FixedSizeBinary(_), DataType::FixedSizeBinary(_)) => { + let left = left + .as_any() + .downcast_ref::() + .unwrap(); + let right = right + .as_any() + .downcast_ref::() + .unwrap(); + Ok(Arc::new(concat_elements_fixed_size_binary(left, right)?)) } // unimplemented _ => Err(ArrowError::NotYetImplemented(format!( @@ -213,6 +359,8 @@ pub fn concat_elements_dyn(left: &dyn Array, right: &dyn Array) -> Result>); + let right = BinaryViewArray::from_iter(vec![] as Vec>); + + let output = concat_elements_binary_view_array(&left, &right).unwrap(); + let expected = BinaryViewArray::from_iter(vec![] as Vec>); + assert_eq!(output, expected); + } + #[test] fn test_concat_dyn_same_type() { // test for StringArray @@ -398,6 +661,26 @@ mod tests { .into(); let expected = LargeBinaryArray::from_opt_vec(vec![None, Some(b"baryyy"), None]); assert_eq!(output, expected); + + // test for BinaryViewArray + let left = BinaryViewArray::from_iter(vec![Some(b"foo" as &[u8]), Some(b"bar"), None]); + let right = BinaryViewArray::from_iter(vec![None, Some(b"yyy" as &[u8]), Some(b"zzz")]); + let output: BinaryViewArray = concat_elements_dyn(&left, &right) + .unwrap() + .into_data() + .into(); + let expected = BinaryViewArray::from_iter(vec![None, Some(b"baryyy" as &[u8]), None]); + assert_eq!(output, expected); + + // test for FixedSizeBinaryArray + let left = FixedSizeBinaryArray::from(vec![Some(b"foo" as &[u8]), Some(b"bar"), None]); + let right = FixedSizeBinaryArray::from(vec![None, Some(b"yyy" as &[u8]), Some(b"zzz")]); + let output: FixedSizeBinaryArray = concat_elements_dyn(&left, &right) + .unwrap() + .into_data() + .into(); + let expected = FixedSizeBinaryArray::from(vec![None, Some(b"baryyy" as &[u8]), None]); + assert_eq!(output, expected); } #[test] From cc5a25649d38f94dbaa6ad9994b6af812d061803 Mon Sep 17 00:00:00 2001 From: Konstantin Tarasov <33369833+sdf-jkl@users.noreply.github.com> Date: Thu, 7 May 2026 08:38:21 -0400 Subject: [PATCH 16/33] impl `FromStr` for `DatePart` (#9931) # Which issue does this PR close? - Closes #9930. # Rationale for this change Check issue # What changes are included in this PR? impl `FromStr` for `DatePart` # Are these changes tested? Yes, added unit tests # Are there any user-facing changes? No --- arrow-arith/src/temporal.rs | 142 ++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/arrow-arith/src/temporal.rs b/arrow-arith/src/temporal.rs index 869fba29cd15..96a3b43a3811 100644 --- a/arrow-arith/src/temporal.rs +++ b/arrow-arith/src/temporal.rs @@ -17,6 +17,7 @@ //! Defines temporal kernels for time and date related functions. +use std::str::FromStr; use std::sync::Arc; use arrow_array::cast::AsArray; @@ -84,6 +85,59 @@ impl std::fmt::Display for DatePart { } } +/// Parses a string into a [`DatePart`]. +/// +/// Matching is case-insensitive. The accepted names follow PostgreSQL's +/// `EXTRACT` / `date_part` conventions (see +/// ) +/// and are aligned with the alias set accepted by `arrow-cast`'s +/// `IntervalUnit::from_str` (plurals, short forms like `y`, `mon`, `ms`, +/// `us`) so the two parsers round-trip on the names they share. +/// +/// Names that do not map one-to-one onto a [`DatePart`] variant are +/// intentionally not recognized: +/// - `epoch` — a derived computation, not a date part. +/// - `century`, `decade`, `millennium` — no matching [`DatePart`] variant. +/// - `timezone`, `timezone_hour`, `timezone_minute` — not modeled by +/// [`DatePart`]. +/// - `isodow` — PostgreSQL's `isodow` returns `1..=7` (Mon=1), but the +/// closest variant ([`DatePart::DayOfWeekMonday0`]) returns `0..=6`. +/// Accepting it here would silently shift the value range; callers that +/// need the PostgreSQL semantic should map the alias themselves and +/// add `1` to the extracted value. +impl FromStr for DatePart { + type Err = ArrowError; + + fn from_str(s: &str) -> Result { + Ok(match s.to_lowercase().as_str() { + "y" | "yr" | "yrs" | "year" | "years" => Self::Year, + "isoyear" => Self::YearISO, + "qtr" | "quarter" | "quarters" => Self::Quarter, + "mon" | "mons" | "month" | "months" => Self::Month, + "w" | "week" | "weeks" => Self::Week, + "isoweek" => Self::WeekISO, + "d" | "day" | "days" => Self::Day, + "dow" | "dayofweek" => Self::DayOfWeekSunday0, + "doy" | "dayofyear" => Self::DayOfYear, + "h" | "hr" | "hrs" | "hour" | "hours" => Self::Hour, + "m" | "min" | "mins" | "minute" | "minutes" => Self::Minute, + "s" | "sec" | "secs" | "second" | "seconds" => Self::Second, + "ms" | "msec" | "msecs" | "msecond" | "mseconds" | "millisecond" | "milliseconds" => { + Self::Millisecond + } + "us" | "usec" | "usecs" | "usecond" | "useconds" | "microsecond" | "microseconds" => { + Self::Microsecond + } + "nanosecond" | "nanoseconds" => Self::Nanosecond, + _ => { + return Err(ArrowError::InvalidArgumentError(format!( + "Unknown date part: {s}" + ))); + } + }) + } +} + /// Returns function to extract relevant [`DatePart`] from types like a /// [`NaiveDateTime`] or [`DateTime`]. /// @@ -2079,4 +2133,92 @@ mod tests { assert_eq!(1, values.value(0)); assert_eq!(2, values.value(1)); } + + #[test] + fn test_date_part_from_str() { + let cases = [ + ("y", DatePart::Year), + ("yr", DatePart::Year), + ("year", DatePart::Year), + ("years", DatePart::Year), + ("YEAR", DatePart::Year), + ("isoyear", DatePart::YearISO), + ("qtr", DatePart::Quarter), + ("quarter", DatePart::Quarter), + ("quarters", DatePart::Quarter), + ("mon", DatePart::Month), + ("mons", DatePart::Month), + ("month", DatePart::Month), + ("MONTHS", DatePart::Month), + ("w", DatePart::Week), + ("week", DatePart::Week), + ("isoweek", DatePart::WeekISO), + ("d", DatePart::Day), + ("day", DatePart::Day), + ("dow", DatePart::DayOfWeekSunday0), + ("DayOfWeek", DatePart::DayOfWeekSunday0), + ("doy", DatePart::DayOfYear), + ("DayOfYear", DatePart::DayOfYear), + ("h", DatePart::Hour), + ("hr", DatePart::Hour), + ("hour", DatePart::Hour), + // Pin the famous `m`/`mon` ambiguity: `m` resolves to Minute, + // matching `arrow-cast`'s `IntervalUnit::from_str`. Use `mon` + // (or `month`) for Month. + ("m", DatePart::Minute), + ("min", DatePart::Minute), + ("mins", DatePart::Minute), + ("minute", DatePart::Minute), + ("s", DatePart::Second), + ("sec", DatePart::Second), + ("second", DatePart::Second), + ("ms", DatePart::Millisecond), + ("msec", DatePart::Millisecond), + ("mseconds", DatePart::Millisecond), + ("millisecond", DatePart::Millisecond), + ("us", DatePart::Microsecond), + ("usec", DatePart::Microsecond), + ("useconds", DatePart::Microsecond), + ("microsecond", DatePart::Microsecond), + ("nanosecond", DatePart::Nanosecond), + ("nanoseconds", DatePart::Nanosecond), + ]; + for (input, expected) in cases { + assert_eq!( + DatePart::from_str(input).unwrap(), + expected, + "parsing {input}" + ); + } + } + + #[test] + fn test_date_part_from_str_unknown() { + // Names intentionally rejected — see the FromStr doc comment for why. + // `isodow` is here because mapping it to `DayOfWeekMonday0` would + // silently shift the value range vs. PostgreSQL's `isodow` (1..=7). + let unknown = [ + "epoch", + "century", + "decade", + "millennium", + "timezone", + "timezone_hour", + "timezone_minute", + "isodow", + // Whitespace is not trimmed — pin this so the behavior doesn't + // change silently. Callers should trim before parsing. + " year ", + "year ", + "", + "nope", + ]; + for s in unknown { + let err = DatePart::from_str(s).unwrap_err(); + assert!( + matches!(err, ArrowError::InvalidArgumentError(_)), + "expected InvalidArgumentError for {s:?}, got: {err}" + ); + } + } } From 3384f649cc07212631111fd2c7e34da750721ec5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 May 2026 08:38:51 -0400 Subject: [PATCH 17/33] chore(deps): bump actions/labeler from 6.0.1 to 6.1.0 (#9932) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/labeler](https://github.com/actions/labeler) from 6.0.1 to 6.1.0.
Release notes

Sourced from actions/labeler's releases.

v6.1.0

Enhancements

  • Add changed-files-labels-limit and max-files-changed configuration options to cap the number of labels added by @​bluca in actions/labeler#923

Bug Fixes

Dependency Updates

New Contributors

Full Changelog: https://github.com/actions/labeler/compare/v6...v6.1.0

Commits
  • f27b608 chore: upgrade dependencies (@​actions/core, @​actions/github, js-yaml, minimat...
  • c5dadc2 Add 'changed-files-labels-limit' and 'max-files-changed' configs to allow cap...
  • e52e4fb Bump minimatch from 10.0.1 to 10.2.3 (#926)
  • 77a4082 Fix: Preserve manually added labels during workflow run and refine label sync...
  • 25abb3c Improve Labeler Action Documentation and Error Handling for Permissions (#897)
  • 395c8cf Bump brace-expansion from 1.1.11 to 1.1.12 and document breaking changes in v...
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/labeler&package-manager=github_actions&previous-version=6.0.1&new-version=6.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dev_pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index 8bf2ec3ccf79..1aa069bd4b22 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -44,7 +44,7 @@ jobs: github.event_name == 'pull_request_target' && (github.event.action == 'opened' || github.event.action == 'synchronize') - uses: actions/labeler@v6.0.1 + uses: actions/labeler@v6.1.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} configuration-path: .github/workflows/dev_pr/labeler.yml From 913bab26ba9bed8fc2bc1acda300cc52345b0da1 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 7 May 2026 10:21:39 -0400 Subject: [PATCH 18/33] Prepare for `58.3.0` release (#9893) # Which issue does this PR close? - Part of https://github.com/apache/arrow-rs/issues/9859 # Rationale for this change Even though we just did a release from 58, I want to get a release out that has these changes: - https://github.com/apache/arrow-rs/pull/9872 - https://github.com/apache/arrow-rs/pull/9813 # What changes are included in this PR? 1. Update version to 58.3.0 2. Update CHANGELOG. See Rendered preview here: https://github.com/alamb/arrow-rs/blob/alamb/prepare_58.3.0/CHANGELOG.md # Are these changes tested? By CI # Are there any user-facing changes? yes --- CHANGELOG-old.md | 175 ++++++++++++++++++++++++++ CHANGELOG.md | 208 ++++++++----------------------- Cargo.toml | 42 +++---- dev/release/update_change_log.sh | 4 +- 4 files changed, 249 insertions(+), 180 deletions(-) diff --git a/CHANGELOG-old.md b/CHANGELOG-old.md index c400a53c03db..0fab24890847 100644 --- a/CHANGELOG-old.md +++ b/CHANGELOG-old.md @@ -21,6 +21,181 @@ # Changelog +## [58.2.0](https://github.com/apache/arrow-rs/tree/58.2.0) (2026-04-28) + +[Full Changelog](https://github.com/apache/arrow-rs/compare/58.1.0...58.2.0) + +**Implemented enhancements:** + +- Expose ColumnCloseResult on ArrowColumnChunk [\#9774](https://github.com/apache/arrow-rs/issues/9774) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Expose FFI data structures fields [\#9771](https://github.com/apache/arrow-rs/issues/9771) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- short-circuit last predicate in `RowFilter` when `with_limit(N)` is set [\#9765](https://github.com/apache/arrow-rs/issues/9765) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- vectorise dict-index bounds check [\#9747](https://github.com/apache/arrow-rs/issues/9747) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Refactor `RleEncoder::flush_bit_packed_run` [\#9734](https://github.com/apache/arrow-rs/issues/9734) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Add benchmark for cast from/to decimals [\#9728](https://github.com/apache/arrow-rs/issues/9728) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Add a security policy for arrow-rs [\#9727](https://github.com/apache/arrow-rs/issues/9727) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] [[arrow-flight](https://github.com/apache/arrow-rs/labels/arrow-flight)] +- Support `FixedSizeList` in arrow-json reader [\#9714](https://github.com/apache/arrow-rs/issues/9714) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[Variant\] Add `VariantArrayBuilder::append_nulls` API [\#9684](https://github.com/apache/arrow-rs/issues/9684) +- \[Json\] RunEndEncoded decoder optimization [\#9645](https://github.com/apache/arrow-rs/issues/9645) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[Variant\] `variant_get(..., List<_>)` non-Struct types support [\#9615](https://github.com/apache/arrow-rs/issues/9615) +- \[Variant\] Add unshredded `Struct` fast-path for `variant_get(..., Struct)` [\#9596](https://github.com/apache/arrow-rs/issues/9596) +- Allow setting custom line terminator for CSV writer [\#9571](https://github.com/apache/arrow-rs/issues/9571) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[Variant\] Align cast logic for `variant_get` to cast kernel for numeric/bool types [\#9564](https://github.com/apache/arrow-rs/issues/9564) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- ci: use ubuntu-slim where applicable [\#9536](https://github.com/apache/arrow-rs/issues/9536) +- Publicly export `arrow_string::Predicate` and its methods? [\#9480](https://github.com/apache/arrow-rs/issues/9480) +- Don't create CompressionContext when no compression is selected \[IPC\] [\#9463](https://github.com/apache/arrow-rs/issues/9463) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Parquet: Raw level buffering causes unbounded memory growth for sparse columns [\#9446](https://github.com/apache/arrow-rs/issues/9446) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Parallel Parquet Reading [\#9381](https://github.com/apache/arrow-rs/issues/9381) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] + +**Fixed bugs:** + +- \[Variant\] `unshred_variant` panics on malformed bytes despite returning `Result` [\#9740](https://github.com/apache/arrow-rs/issues/9740) +- RecordBatch::normalize\(\) does not propagate top level null bitmap into the results [\#9732](https://github.com/apache/arrow-rs/issues/9732) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Incorrect accounting in `DictEncoder::estimated_memory_size` [\#9719](https://github.com/apache/arrow-rs/issues/9719) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- arrow-ipc writer does not comply with spec for empty variable-size arrays [\#9716](https://github.com/apache/arrow-rs/issues/9716) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Panic when reading corrupt parquet file with truncated data instead of ParquetError [\#9705](https://github.com/apache/arrow-rs/issues/9705) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- NOTICE.txt is inaccurate [\#9703](https://github.com/apache/arrow-rs/issues/9703) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Unnecessary dependency on regex crate [\#9672](https://github.com/apache/arrow-rs/issues/9672) +- \[arrow-avro\] Avro reader produces incorrect results when reader schema and writer schema differ [\#9655](https://github.com/apache/arrow-rs/issues/9655) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- parquet docs are broken on docs.rs [\#9649](https://github.com/apache/arrow-rs/issues/9649) +- \[Parquet\] ArrowWriter with CDC panics on nested ListArrays [\#9637](https://github.com/apache/arrow-rs/issues/9637) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] [[arrow-flight](https://github.com/apache/arrow-rs/labels/arrow-flight)] +- Use release KEYS file for verification instead of dev KEYS [\#9603](https://github.com/apache/arrow-rs/issues/9603) +- IPC reader: handling of dictionaries with only null values [\#9595](https://github.com/apache/arrow-rs/issues/9595) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Parquet RleDecoder::get\_batch\_with\_dict panics on oob dictionary indices [\#9434](https://github.com/apache/arrow-rs/issues/9434) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] + +**Documentation updates:** + +- docs\(variant\): link VariantArray doc to official Parquet Variant extension type [\#9779](https://github.com/apache/arrow-rs/pull/9779) ([mcharrel](https://github.com/mcharrel)) +- Document Security Policy [\#9730](https://github.com/apache/arrow-rs/pull/9730) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] [[arrow-flight](https://github.com/apache/arrow-rs/labels/arrow-flight)] ([alamb](https://github.com/alamb)) +- Docs: add example of how to read parquet row groups in parallel [\#9396](https://github.com/apache/arrow-rs/pull/9396) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([alamb](https://github.com/alamb)) + +**Performance improvements:** + +- parquet: avoid decode and heap allocation on terminal skip in DeltaBitPackDecoder [\#9784](https://github.com/apache/arrow-rs/issues/9784) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- parquet: O\(1\) skip for bw=0 miniblocks in DeltaBitPackDecoder [\#9783](https://github.com/apache/arrow-rs/issues/9783) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Remove per-message flush overhead in Arrow IPC writer [\#9762](https://github.com/apache/arrow-rs/issues/9762) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Support `GenericListViewArray::new_unchecked` and refactor ListView json decoder [\#9646](https://github.com/apache/arrow-rs/issues/9646) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Support nested REE in arrow-ord `partition` function [\#9640](https://github.com/apache/arrow-rs/issues/9640) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[Parquet\] Remove the BIT\_PACKED encoder [\#9635](https://github.com/apache/arrow-rs/issues/9635) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Pre-reserve output capacity in ByteView/ByteArray dictionary decoding [\#9587](https://github.com/apache/arrow-rs/issues/9587) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Fuse RLE decoding and view gathering for StringView dictionary decoding [\#9582](https://github.com/apache/arrow-rs/issues/9582) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Use branchless index clamping and add get\_batch\_direct to RleDecoder [\#9581](https://github.com/apache/arrow-rs/issues/9581) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Reduce per-byte overhead in VLQ integer decoding [\#9580](https://github.com/apache/arrow-rs/issues/9580) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- feat\(parquet\): batch RLE runs in level encoder via scan-ahead [\#9830](https://github.com/apache/arrow-rs/pull/9830) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- fix: lazy-init zstd compression contexts to avoid unnecessary FFI calls [\#9808](https://github.com/apache/arrow-rs/pull/9808) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mbutrovich](https://github.com/mbutrovich)) +- parquet: O\(1\) skip for bw=0 miniblocks in DeltaBitPackDecoder [\#9786](https://github.com/apache/arrow-rs/pull/9786) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([sahuagin](https://github.com/sahuagin)) +- chore: add benchmark for row filters with LIMIT short-circuit [\#9767](https://github.com/apache/arrow-rs/pull/9767) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([haohuaijin](https://github.com/haohuaijin)) +- Push `LIMIT` / `OFFSET` into the last `RowFilter` predicate and skip unused row groups [\#9766](https://github.com/apache/arrow-rs/pull/9766) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([haohuaijin](https://github.com/haohuaijin)) +- feat\(ipc\): Remove per-message flush in IPC writer hot path [\#9763](https://github.com/apache/arrow-rs/pull/9763) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pchintar](https://github.com/pchintar)) +- perf\(parquet\): Defer fixed length byte array buffer alloc and skip zero-batch init [\#9756](https://github.com/apache/arrow-rs/pull/9756) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([lyang24](https://github.com/lyang24)) +- feat\(parquet\): batch consecutive null/empty rows in `write_list` [\#9752](https://github.com/apache/arrow-rs/pull/9752) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- Remove `len` field from buffer builder [\#9750](https://github.com/apache/arrow-rs/pull/9750) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([cetra3](https://github.com/cetra3)) +- perf\(parquet\): Vectorize dict-index bounds check in RleDecoder::get\_batch\_with\_dict \(up to -7.9%\) [\#9746](https://github.com/apache/arrow-rs/pull/9746) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) +- feat\(parquet\): precompute `offset_index_disabled` at build-time [\#9724](https://github.com/apache/arrow-rs/pull/9724) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- \[Parquet\] Improve dictionary decoder by unrolling loops [\#9662](https://github.com/apache/arrow-rs/pull/9662) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) +- \[Json\] Use `partition` and `take` in RunEndEncoded decoder [\#9658](https://github.com/apache/arrow-rs/pull/9658) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- Improve take performance on List arrays [\#9643](https://github.com/apache/arrow-rs/pull/9643) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([AdamGS](https://github.com/AdamGS)) +- \[Json\] Replace `ArrayData` with typed Array construction in json-reader [\#9497](https://github.com/apache/arrow-rs/pull/9497) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- feat\(parquet\): stream-encode definition/repetition levels incrementally [\#9447](https://github.com/apache/arrow-rs/pull/9447) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) + +**Closed issues:** + +- Incorrect buffer skipping for V4 Union types in IPC `skip_field` [\#9828](https://github.com/apache/arrow-rs/issues/9828) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Replace wildcard match in `skip_field` with explicit `DataType` handling [\#9821](https://github.com/apache/arrow-rs/issues/9821) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Column projection misalignment for ListView / LargeListView in IPC reader [\#9805](https://github.com/apache/arrow-rs/issues/9805) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Avoid panic on malformed compressed buffer prefix in IPC [\#9801](https://github.com/apache/arrow-rs/issues/9801) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- DeltaByteArrayDecoder panics on invalid prefix lengths [\#9796](https://github.com/apache/arrow-rs/issues/9796) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Use NullBufferBuilder when reading json [\#9781](https://github.com/apache/arrow-rs/issues/9781) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Perfectly shredded arrays with top-level null values loss nullability when `typed_value` is extracted [\#9701](https://github.com/apache/arrow-rs/issues/9701) +- \[Parquet Metadata\] API to determine page-index presence separately from page-index load [\#9693](https://github.com/apache/arrow-rs/issues/9693) +- Union cast is incorrect for duplicate field names [\#9664](https://github.com/apache/arrow-rs/issues/9664) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- List and ListView are missing `take` benchmarks [\#9627](https://github.com/apache/arrow-rs/issues/9627) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Support RunEndEncoded arrays in comparison kernels \(eq, lt, etc.\) [\#9620](https://github.com/apache/arrow-rs/issues/9620) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- variant\_get should follow JSONpath semantics [\#9606](https://github.com/apache/arrow-rs/issues/9606) +- GenericByteViewArray: support finding total length of all strings [\#9435](https://github.com/apache/arrow-rs/issues/9435) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] + +**Merged pull requests:** + +- support length\(\) on Run-end encoding arrays [\#9838](https://github.com/apache/arrow-rs/pull/9838) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Rich-T-kid](https://github.com/Rich-T-kid)) +- fix\(ipc\): correct skip\_field handling for V4 Union [\#9829](https://github.com/apache/arrow-rs/pull/9829) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pchintar](https://github.com/pchintar)) +- fix\(ipc\): replace wildcard in skip\_field with explicit DataType handling [\#9822](https://github.com/apache/arrow-rs/pull/9822) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pchintar](https://github.com/pchintar)) +- Prevent buffer builder length overflow in `MutableBuffer::extend_zeros` [\#9820](https://github.com/apache/arrow-rs/pull/9820) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Prevent repeat slice length overflow [\#9819](https://github.com/apache/arrow-rs/pull/9819) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Prevent BitChunks length overflow [\#9818](https://github.com/apache/arrow-rs/pull/9818) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Prevent Rows row index overflow [\#9817](https://github.com/apache/arrow-rs/pull/9817) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Prevent ArrayData validation length overflow [\#9816](https://github.com/apache/arrow-rs/pull/9816) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- \[Json\] Remove arrow-data dependency from arrow-json [\#9812](https://github.com/apache/arrow-rs/pull/9812) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- Replace `BooleanBufferBuilder` with `NullBufferBuilder` in arrow-json if applicable [\#9811](https://github.com/apache/arrow-rs/pull/9811) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- refactor\(ipc\): derive Default for CompressionContext [\#9809](https://github.com/apache/arrow-rs/pull/9809) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mbutrovich](https://github.com/mbutrovich)) +- fix\(ipc\): reader misalignment when skipping ListView / LargeListView columns [\#9806](https://github.com/apache/arrow-rs/pull/9806) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pchintar](https://github.com/pchintar)) +- fix\(ipc\): Avoid panic on malformed compressed buffer prefix [\#9802](https://github.com/apache/arrow-rs/pull/9802) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pchintar](https://github.com/pchintar)) +- parquet: fix panic in DeltaByteArrayDecoder on invalid prefix lengths [\#9797](https://github.com/apache/arrow-rs/pull/9797) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([pchintar](https://github.com/pchintar)) +- feat\(parquet\): fuse level encoding with counting and histogram updates [\#9795](https://github.com/apache/arrow-rs/pull/9795) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- Expose ColumnCloseResult on ArrowColumnChunk [\#9773](https://github.com/apache/arrow-rs/pull/9773) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([leoyvens](https://github.com/leoyvens)) +- feat: make FFI structs fields `pub` [\#9772](https://github.com/apache/arrow-rs/pull/9772) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([ashdnazg](https://github.com/ashdnazg)) +- chore: Refine the error message for List to non List cast [\#9757](https://github.com/apache/arrow-rs/pull/9757) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([comphead](https://github.com/comphead)) +- refactor\(parquet\): replace magic `8` literals with named constants [\#9751](https://github.com/apache/arrow-rs/pull/9751) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- feat\(ipc\): add with\_skip\_validation to StreamDecoder [\#9749](https://github.com/apache/arrow-rs/pull/9749) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pantShrey](https://github.com/pantShrey)) +- remove panics in unshred variant [\#9741](https://github.com/apache/arrow-rs/pull/9741) ([friendlymatthew](https://github.com/friendlymatthew)) +- Add benchmark for ListView interleave [\#9738](https://github.com/apache/arrow-rs/pull/9738) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([vegarsti](https://github.com/vegarsti)) +- arrow-arith: fix 'occured' -\> 'occurred' in arity.rs comments [\#9736](https://github.com/apache/arrow-rs/pull/9736) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([SAY-5](https://github.com/SAY-5)) +- Refactor `RleEncoder::flush_bit_packed_run` to make flow clearer [\#9735](https://github.com/apache/arrow-rs/pull/9735) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://github.com/etseidl)) +- Fix RecordBatch::normalize\(\) null bitmap bug and add StructArray::flatten\(\) [\#9733](https://github.com/apache/arrow-rs/pull/9733) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([sqd](https://github.com/sqd)) +- Add benchmark for cast from/to decimals [\#9729](https://github.com/apache/arrow-rs/pull/9729) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([klion26](https://github.com/klion26)) +- refactor\(arrow-avro\): use `Decoder::flush_block` in async reader [\#9726](https://github.com/apache/arrow-rs/pull/9726) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mzabaluev](https://github.com/mzabaluev)) +- fix: ParquetError when reading corrupt parquet file with truncated data instead of Panic [\#9725](https://github.com/apache/arrow-rs/pull/9725) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([xuzifu666](https://github.com/xuzifu666)) +- feat\(parquet\): add wide-schema writer overhead benchmark [\#9723](https://github.com/apache/arrow-rs/pull/9723) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- fix: correct accounting in `DictEncoder::estimated_memory_size`, `Interner::estimated_memory_size` [\#9720](https://github.com/apache/arrow-rs/pull/9720) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([mzabaluev](https://github.com/mzabaluev)) +- arrow-ipc: Write 0 offset buffer for length-0 variable-size arrays [\#9717](https://github.com/apache/arrow-rs/pull/9717) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([atwam](https://github.com/atwam)) +- \[Json\] Support `FixedSizeList` in json decoder [\#9715](https://github.com/apache/arrow-rs/pull/9715) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- chore\(deps\): bump actions/upload-pages-artifact from 4 to 5 [\#9713](https://github.com/apache/arrow-rs/pull/9713) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Fix clippy warning in fixed\_size\_binary\_array.rs [\#9712](https://github.com/apache/arrow-rs/pull/9712) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([AdamGS](https://github.com/AdamGS)) +- feat: add `has_non_empty_nulls` helper function in `OffsetBuffer` [\#9711](https://github.com/apache/arrow-rs/pull/9711) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) +- chore\(deps\): bump pytest from 7.2.0 to 9.0.3 in /parquet/pytest [\#9706](https://github.com/apache/arrow-rs/pull/9706) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([dependabot[bot]](https://github.com/apps/dependabot)) +- Fedora license audit [\#9704](https://github.com/apache/arrow-rs/pull/9704) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([michel-slm](https://github.com/michel-slm)) +- \[Variant\] Take top-level nulls into consideration when extracting perfectly shredded children [\#9702](https://github.com/apache/arrow-rs/pull/9702) ([AdamGS](https://github.com/AdamGS)) +- feat\(parquet\): add `push_decoder` benchmark for `PushBuffers` overhead [\#9696](https://github.com/apache/arrow-rs/pull/9696) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- Add mutable bitwise operations to `BooleanArray` and `NullBuffer::union_many` [\#9692](https://github.com/apache/arrow-rs/pull/9692) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mbutrovich](https://github.com/mbutrovich)) +- chore\(deps\): update hashbrown requirement from 0.16.0 to 0.17.0 [\#9691](https://github.com/apache/arrow-rs/pull/9691) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([dependabot[bot]](https://github.com/apps/dependabot)) +- chore\(deps\): bump actions/github-script from 8 to 9 [\#9690](https://github.com/apache/arrow-rs/pull/9690) ([dependabot[bot]](https://github.com/apps/dependabot)) +- minor: Re-enable CDC bench [\#9686](https://github.com/apache/arrow-rs/pull/9686) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://github.com/etseidl)) +- \[Variant\] Add `VariantArrayBuilder::append_nulls` API [\#9685](https://github.com/apache/arrow-rs/pull/9685) ([sdf-jkl](https://github.com/sdf-jkl)) +- feat\(parquet\): add struct-column writer benchmarks [\#9679](https://github.com/apache/arrow-rs/pull/9679) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- \[Arrow\] Add API to check if `Field` has a valid `ExtensionType` [\#9677](https://github.com/apache/arrow-rs/pull/9677) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([sdf-jkl](https://github.com/sdf-jkl)) +- \[Variant\] `variant_get` should follow JSONPath semantics for Field path element [\#9676](https://github.com/apache/arrow-rs/pull/9676) ([sdf-jkl](https://github.com/sdf-jkl)) +- ParquetMetaDataPushDecoder API to clear all buffered ranges [\#9673](https://github.com/apache/arrow-rs/pull/9673) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([nathanb9](https://github.com/nathanb9)) +- Fix union cast incorrectness for duplicate field names [\#9666](https://github.com/apache/arrow-rs/pull/9666) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([friendlymatthew](https://github.com/friendlymatthew)) +- chore: re-export `MAX_INLINE_VIEW_LEN` from `arrow_data` [\#9665](https://github.com/apache/arrow-rs/pull/9665) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) +- No longer allow BIT\_PACKED level encoding in Parquet writer [\#9656](https://github.com/apache/arrow-rs/pull/9656) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://github.com/etseidl)) +- feat\(parquet\): add sparse-column writer benchmarks [\#9654](https://github.com/apache/arrow-rs/pull/9654) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- Support `GenericListViewArray::new_unchecked` and refactor `ListView` json decoder [\#9648](https://github.com/apache/arrow-rs/pull/9648) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- \[Json\] Add json reader benchmarks for ListView [\#9647](https://github.com/apache/arrow-rs/pull/9647) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- fix\(parquet\): fix CDC panic on nested ListArrays with null entries [\#9644](https://github.com/apache/arrow-rs/pull/9644) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([kszucs](https://github.com/kszucs)) +- Add a test for reading nested REE data in json [\#9634](https://github.com/apache/arrow-rs/pull/9634) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- \[Variant\] Fix `variant_get` to return `List` instead of `List` [\#9631](https://github.com/apache/arrow-rs/pull/9631) ([liamzwbao](https://github.com/liamzwbao)) +- ci: use ubuntu-slim runner for lightweight CI jobs [\#9630](https://github.com/apache/arrow-rs/pull/9630) ([CuteChuanChuan](https://github.com/CuteChuanChuan)) +- Add bloom filter folding to automatically size SBBF filters [\#9628](https://github.com/apache/arrow-rs/pull/9628) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([adriangb](https://github.com/adriangb)) +- Add List and ListView take benchmarks [\#9626](https://github.com/apache/arrow-rs/pull/9626) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([AdamGS](https://github.com/AdamGS)) +- ParquetPushDecoder API to clear all buffered ranges [\#9624](https://github.com/apache/arrow-rs/pull/9624) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([nathanb9](https://github.com/nathanb9)) +- fix: handle missing dictionary batch for null-only columns in IPC reader [\#9623](https://github.com/apache/arrow-rs/pull/9623) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([joaquinhuigomez](https://github.com/joaquinhuigomez)) +- Fix `MutableBuffer::clear` [\#9622](https://github.com/apache/arrow-rs/pull/9622) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Rafferty97](https://github.com/Rafferty97)) +- feat\[arrow-ord\]: suppport REE comparisons [\#9621](https://github.com/apache/arrow-rs/pull/9621) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([asubiotto](https://github.com/asubiotto)) +- chore\(deps\): update sha2 requirement from 0.10 to 0.11 [\#9618](https://github.com/apache/arrow-rs/pull/9618) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([dependabot[bot]](https://github.com/apps/dependabot)) +- Expose option to set line terminator for CSV writer [\#9617](https://github.com/apache/arrow-rs/pull/9617) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([svranesevic](https://github.com/svranesevic)) +- \[Json\] Add json reader benchmarks for Map and REE [\#9616](https://github.com/apache/arrow-rs/pull/9616) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- deps: fix `object_store` breakage for 0.13.2 [\#9612](https://github.com/apache/arrow-rs/pull/9612) ([mzabaluev-flarion](https://github.com/mzabaluev-flarion)) +- \[Variant\] Support Binary/LargeBinary children [\#9610](https://github.com/apache/arrow-rs/pull/9610) ([AdamGS](https://github.com/AdamGS)) +- fix: use writer types in Skipper for resolved named record types [\#9605](https://github.com/apache/arrow-rs/pull/9605) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([ariel-miculas](https://github.com/ariel-miculas)) +- feat\(parquet\): derive `PartialEq` and `Eq` for `CdcOptions` [\#9602](https://github.com/apache/arrow-rs/pull/9602) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([kszucs](https://github.com/kszucs)) +- Add `finish_preserve_values` to `ArrayBuilder` trait [\#9601](https://github.com/apache/arrow-rs/pull/9601) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([adamreichold](https://github.com/adamreichold)) +- \[Variant\] extend shredded null handling for arrays [\#9599](https://github.com/apache/arrow-rs/pull/9599) ([sdf-jkl](https://github.com/sdf-jkl)) +- \[Variant\] Add unshredded `Struct` fast-path for `variant_get(..., Struct)` [\#9597](https://github.com/apache/arrow-rs/pull/9597) ([sdf-jkl](https://github.com/sdf-jkl)) +- Pre-reserve output capacity in ByteView/ByteArray dictionary decoding [\#9590](https://github.com/apache/arrow-rs/pull/9590) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) +- \[Variant\] Align cast logic for variant\_get to cast kernel for numeric/bool types [\#9563](https://github.com/apache/arrow-rs/pull/9563) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([klion26](https://github.com/klion26)) +- Add support to cast from `UnionArray` [\#9544](https://github.com/apache/arrow-rs/pull/9544) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([friendlymatthew](https://github.com/friendlymatthew)) +- Support `ListView` codec in arrow-json [\#9503](https://github.com/apache/arrow-rs/pull/9503) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) + + ## [58.1.0](https://github.com/apache/arrow-rs/tree/58.1.0) (2026-03-20) [Full Changelog](https://github.com/apache/arrow-rs/compare/58.0.0...58.1.0) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03c4e1579446..311d5e99a368 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,179 +19,73 @@ # Changelog -## [58.2.0](https://github.com/apache/arrow-rs/tree/58.2.0) (2026-04-28) +## [58.3.0](https://github.com/apache/arrow-rs/tree/58.3.0) (2026-05-07) -[Full Changelog](https://github.com/apache/arrow-rs/compare/58.1.0...58.2.0) +[Full Changelog](https://github.com/apache/arrow-rs/compare/58.2.0...58.3.0) **Implemented enhancements:** -- Expose ColumnCloseResult on ArrowColumnChunk [\#9774](https://github.com/apache/arrow-rs/issues/9774) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- Expose FFI data structures fields [\#9771](https://github.com/apache/arrow-rs/issues/9771) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- short-circuit last predicate in `RowFilter` when `with_limit(N)` is set [\#9765](https://github.com/apache/arrow-rs/issues/9765) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- vectorise dict-index bounds check [\#9747](https://github.com/apache/arrow-rs/issues/9747) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- Refactor `RleEncoder::flush_bit_packed_run` [\#9734](https://github.com/apache/arrow-rs/issues/9734) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- Add benchmark for cast from/to decimals [\#9728](https://github.com/apache/arrow-rs/issues/9728) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Add a security policy for arrow-rs [\#9727](https://github.com/apache/arrow-rs/issues/9727) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] [[arrow-flight](https://github.com/apache/arrow-rs/labels/arrow-flight)] -- Support `FixedSizeList` in arrow-json reader [\#9714](https://github.com/apache/arrow-rs/issues/9714) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- \[Variant\] Add `VariantArrayBuilder::append_nulls` API [\#9684](https://github.com/apache/arrow-rs/issues/9684) -- \[Json\] RunEndEncoded decoder optimization [\#9645](https://github.com/apache/arrow-rs/issues/9645) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- \[Variant\] `variant_get(..., List<_>)` non-Struct types support [\#9615](https://github.com/apache/arrow-rs/issues/9615) -- \[Variant\] Add unshredded `Struct` fast-path for `variant_get(..., Struct)` [\#9596](https://github.com/apache/arrow-rs/issues/9596) -- Allow setting custom line terminator for CSV writer [\#9571](https://github.com/apache/arrow-rs/issues/9571) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- \[Variant\] Align cast logic for `variant_get` to cast kernel for numeric/bool types [\#9564](https://github.com/apache/arrow-rs/issues/9564) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- ci: use ubuntu-slim where applicable [\#9536](https://github.com/apache/arrow-rs/issues/9536) -- Publicly export `arrow_string::Predicate` and its methods? [\#9480](https://github.com/apache/arrow-rs/issues/9480) -- Don't create CompressionContext when no compression is selected \[IPC\] [\#9463](https://github.com/apache/arrow-rs/issues/9463) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Parquet: Raw level buffering causes unbounded memory growth for sparse columns [\#9446](https://github.com/apache/arrow-rs/issues/9446) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- Parallel Parquet Reading [\#9381](https://github.com/apache/arrow-rs/issues/9381) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Add `DatePart::from_str` API [\#9930](https://github.com/apache/arrow-rs/issues/9930) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- should use DictionaryArray::with\_values instead of try\_new on the dictionary fast path [\#9889](https://github.com/apache/arrow-rs/issues/9889) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[arrow-string\] add concat\_elements for BinaryViewArray and FixedSizeBinary [\#9875](https://github.com/apache/arrow-rs/issues/9875) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Expose eq ignore ascii case from arrow-string [\#9870](https://github.com/apache/arrow-rs/issues/9870) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Configurable data page v2 compression threshold [\#9827](https://github.com/apache/arrow-rs/issues/9827) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] **Fixed bugs:** -- \[Variant\] `unshred_variant` panics on malformed bytes despite returning `Result` [\#9740](https://github.com/apache/arrow-rs/issues/9740) -- RecordBatch::normalize\(\) does not propagate top level null bitmap into the results [\#9732](https://github.com/apache/arrow-rs/issues/9732) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Incorrect accounting in `DictEncoder::estimated_memory_size` [\#9719](https://github.com/apache/arrow-rs/issues/9719) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- arrow-ipc writer does not comply with spec for empty variable-size arrays [\#9716](https://github.com/apache/arrow-rs/issues/9716) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Panic when reading corrupt parquet file with truncated data instead of ParquetError [\#9705](https://github.com/apache/arrow-rs/issues/9705) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- NOTICE.txt is inaccurate [\#9703](https://github.com/apache/arrow-rs/issues/9703) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Unnecessary dependency on regex crate [\#9672](https://github.com/apache/arrow-rs/issues/9672) -- \[arrow-avro\] Avro reader produces incorrect results when reader schema and writer schema differ [\#9655](https://github.com/apache/arrow-rs/issues/9655) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- parquet docs are broken on docs.rs [\#9649](https://github.com/apache/arrow-rs/issues/9649) -- \[Parquet\] ArrowWriter with CDC panics on nested ListArrays [\#9637](https://github.com/apache/arrow-rs/issues/9637) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] [[arrow-flight](https://github.com/apache/arrow-rs/labels/arrow-flight)] -- Use release KEYS file for verification instead of dev KEYS [\#9603](https://github.com/apache/arrow-rs/issues/9603) -- IPC reader: handling of dictionaries with only null values [\#9595](https://github.com/apache/arrow-rs/issues/9595) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Parquet RleDecoder::get\_batch\_with\_dict panics on oob dictionary indices [\#9434](https://github.com/apache/arrow-rs/issues/9434) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- \[arrow-cast\] incorrect Time32 -\> Time64 conversion [\#9851](https://github.com/apache/arrow-rs/issues/9851) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Panic when reading malformed compact-Thrift bool fields in Parquet page metadata [\#9839](https://github.com/apache/arrow-rs/issues/9839) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Parquet `DeltaBitPackDecoder::skip` could panic on "non-standard" miniblocks [\#9793](https://github.com/apache/arrow-rs/issues/9793) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] **Documentation updates:** -- docs\(variant\): link VariantArray doc to official Parquet Variant extension type [\#9779](https://github.com/apache/arrow-rs/pull/9779) ([mcharrel](https://github.com/mcharrel)) -- Document Security Policy [\#9730](https://github.com/apache/arrow-rs/pull/9730) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] [[arrow-flight](https://github.com/apache/arrow-rs/labels/arrow-flight)] ([alamb](https://github.com/alamb)) -- Docs: add example of how to read parquet row groups in parallel [\#9396](https://github.com/apache/arrow-rs/pull/9396) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([alamb](https://github.com/alamb)) - -**Performance improvements:** - -- parquet: avoid decode and heap allocation on terminal skip in DeltaBitPackDecoder [\#9784](https://github.com/apache/arrow-rs/issues/9784) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- parquet: O\(1\) skip for bw=0 miniblocks in DeltaBitPackDecoder [\#9783](https://github.com/apache/arrow-rs/issues/9783) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- Remove per-message flush overhead in Arrow IPC writer [\#9762](https://github.com/apache/arrow-rs/issues/9762) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Support `GenericListViewArray::new_unchecked` and refactor ListView json decoder [\#9646](https://github.com/apache/arrow-rs/issues/9646) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Support nested REE in arrow-ord `partition` function [\#9640](https://github.com/apache/arrow-rs/issues/9640) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- \[Parquet\] Remove the BIT\_PACKED encoder [\#9635](https://github.com/apache/arrow-rs/issues/9635) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- Pre-reserve output capacity in ByteView/ByteArray dictionary decoding [\#9587](https://github.com/apache/arrow-rs/issues/9587) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- Fuse RLE decoding and view gathering for StringView dictionary decoding [\#9582](https://github.com/apache/arrow-rs/issues/9582) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- Use branchless index clamping and add get\_batch\_direct to RleDecoder [\#9581](https://github.com/apache/arrow-rs/issues/9581) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- Reduce per-byte overhead in VLQ integer decoding [\#9580](https://github.com/apache/arrow-rs/issues/9580) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- feat\(parquet\): batch RLE runs in level encoder via scan-ahead [\#9830](https://github.com/apache/arrow-rs/pull/9830) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) -- fix: lazy-init zstd compression contexts to avoid unnecessary FFI calls [\#9808](https://github.com/apache/arrow-rs/pull/9808) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mbutrovich](https://github.com/mbutrovich)) -- parquet: O\(1\) skip for bw=0 miniblocks in DeltaBitPackDecoder [\#9786](https://github.com/apache/arrow-rs/pull/9786) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([sahuagin](https://github.com/sahuagin)) -- chore: add benchmark for row filters with LIMIT short-circuit [\#9767](https://github.com/apache/arrow-rs/pull/9767) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([haohuaijin](https://github.com/haohuaijin)) -- Push `LIMIT` / `OFFSET` into the last `RowFilter` predicate and skip unused row groups [\#9766](https://github.com/apache/arrow-rs/pull/9766) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([haohuaijin](https://github.com/haohuaijin)) -- feat\(ipc\): Remove per-message flush in IPC writer hot path [\#9763](https://github.com/apache/arrow-rs/pull/9763) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pchintar](https://github.com/pchintar)) -- perf\(parquet\): Defer fixed length byte array buffer alloc and skip zero-batch init [\#9756](https://github.com/apache/arrow-rs/pull/9756) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([lyang24](https://github.com/lyang24)) -- feat\(parquet\): batch consecutive null/empty rows in `write_list` [\#9752](https://github.com/apache/arrow-rs/pull/9752) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) -- Remove `len` field from buffer builder [\#9750](https://github.com/apache/arrow-rs/pull/9750) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([cetra3](https://github.com/cetra3)) -- perf\(parquet\): Vectorize dict-index bounds check in RleDecoder::get\_batch\_with\_dict \(up to -7.9%\) [\#9746](https://github.com/apache/arrow-rs/pull/9746) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) -- feat\(parquet\): precompute `offset_index_disabled` at build-time [\#9724](https://github.com/apache/arrow-rs/pull/9724) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) -- \[Parquet\] Improve dictionary decoder by unrolling loops [\#9662](https://github.com/apache/arrow-rs/pull/9662) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) -- \[Json\] Use `partition` and `take` in RunEndEncoded decoder [\#9658](https://github.com/apache/arrow-rs/pull/9658) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) -- Improve take performance on List arrays [\#9643](https://github.com/apache/arrow-rs/pull/9643) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([AdamGS](https://github.com/AdamGS)) -- \[Json\] Replace `ArrayData` with typed Array construction in json-reader [\#9497](https://github.com/apache/arrow-rs/pull/9497) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) -- feat\(parquet\): stream-encode definition/repetition levels incrementally [\#9447](https://github.com/apache/arrow-rs/pull/9447) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- docs: Add guidance for AI assisted submissions to CONTRIBUTING.md [\#9892](https://github.com/apache/arrow-rs/pull/9892) ([etseidl](https://github.com/etseidl)) +- Update release schedule on README [\#9881](https://github.com/apache/arrow-rs/pull/9881) ([alamb](https://github.com/alamb)) +- Add more documentation for FixedSizeBinary arrays [\#9866](https://github.com/apache/arrow-rs/pull/9866) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Minor: document why FixedSizeBinary offset is always 0 [\#9861](https://github.com/apache/arrow-rs/pull/9861) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- docs: Update contributing guidelines with benchmark results [\#9782](https://github.com/apache/arrow-rs/pull/9782) ([alamb](https://github.com/alamb)) **Closed issues:** -- Incorrect buffer skipping for V4 Union types in IPC `skip_field` [\#9828](https://github.com/apache/arrow-rs/issues/9828) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Replace wildcard match in `skip_field` with explicit `DataType` handling [\#9821](https://github.com/apache/arrow-rs/issues/9821) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Column projection misalignment for ListView / LargeListView in IPC reader [\#9805](https://github.com/apache/arrow-rs/issues/9805) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Avoid panic on malformed compressed buffer prefix in IPC [\#9801](https://github.com/apache/arrow-rs/issues/9801) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- DeltaByteArrayDecoder panics on invalid prefix lengths [\#9796](https://github.com/apache/arrow-rs/issues/9796) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- Use NullBufferBuilder when reading json [\#9781](https://github.com/apache/arrow-rs/issues/9781) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Perfectly shredded arrays with top-level null values loss nullability when `typed_value` is extracted [\#9701](https://github.com/apache/arrow-rs/issues/9701) -- \[Parquet Metadata\] API to determine page-index presence separately from page-index load [\#9693](https://github.com/apache/arrow-rs/issues/9693) -- Union cast is incorrect for duplicate field names [\#9664](https://github.com/apache/arrow-rs/issues/9664) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- List and ListView are missing `take` benchmarks [\#9627](https://github.com/apache/arrow-rs/issues/9627) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Support RunEndEncoded arrays in comparison kernels \(eq, lt, etc.\) [\#9620](https://github.com/apache/arrow-rs/issues/9620) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- variant\_get should follow JSONpath semantics [\#9606](https://github.com/apache/arrow-rs/issues/9606) -- GenericByteViewArray: support finding total length of all strings [\#9435](https://github.com/apache/arrow-rs/issues/9435) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- GenericByteDictionaryBuilder::with\_capacity does not pre-size dedup HashTable [\#9907](https://github.com/apache/arrow-rs/issues/9907) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[arrow-buffer\] Integer overflow in repeat\_slice\_n\_times leads to undefined behavior [\#9904](https://github.com/apache/arrow-rs/issues/9904) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[arrow-buffer\] Integer overflow in BitChunks::new leads to undefined behavior [\#9903](https://github.com/apache/arrow-rs/issues/9903) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[arrow-row\] Integer overflow in Rows::row index handling leads to undefined behavior [\#9901](https://github.com/apache/arrow-rs/issues/9901) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[arrow-data\] Integer overflow in ArrayData validation leads to undefined behavior [\#9900](https://github.com/apache/arrow-rs/issues/9900) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[arrow-data\] Integer overflow in ArrayData::slice leads to undefined behavior [\#9899](https://github.com/apache/arrow-rs/issues/9899) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[arrow-array\] Integer overflow in FixedSizeBinaryArray::value leads to undefined behavior [\#9898](https://github.com/apache/arrow-rs/issues/9898) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[arrow-buffer\] Integer overflow in BufferBuilder::reserve leads to undefined behavior [\#9897](https://github.com/apache/arrow-rs/issues/9897) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- arrow-csv: integer overflow panic in Reader::records::flush [\#9885](https://github.com/apache/arrow-rs/issues/9885) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Make an API to help with the pattern of 'replaces the values of the REE array' [\#9854](https://github.com/apache/arrow-rs/issues/9854) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Parquet reader rejects canonical UNKNOWN logical type on BOOLEAN physical columns [\#9844](https://github.com/apache/arrow-rs/issues/9844) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- ColumnIndex length mismatch can cause panic during decoding in Parquet [\#9832](https://github.com/apache/arrow-rs/issues/9832) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Bug converting json to fixed list of zero size [\#9780](https://github.com/apache/arrow-rs/issues/9780) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] **Merged pull requests:** -- support length\(\) on Run-end encoding arrays [\#9838](https://github.com/apache/arrow-rs/pull/9838) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Rich-T-kid](https://github.com/Rich-T-kid)) -- fix\(ipc\): correct skip\_field handling for V4 Union [\#9829](https://github.com/apache/arrow-rs/pull/9829) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pchintar](https://github.com/pchintar)) -- fix\(ipc\): replace wildcard in skip\_field with explicit DataType handling [\#9822](https://github.com/apache/arrow-rs/pull/9822) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pchintar](https://github.com/pchintar)) -- Prevent buffer builder length overflow in `MutableBuffer::extend_zeros` [\#9820](https://github.com/apache/arrow-rs/pull/9820) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Prevent repeat slice length overflow [\#9819](https://github.com/apache/arrow-rs/pull/9819) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Prevent BitChunks length overflow [\#9818](https://github.com/apache/arrow-rs/pull/9818) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Prevent Rows row index overflow [\#9817](https://github.com/apache/arrow-rs/pull/9817) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Prevent ArrayData validation length overflow [\#9816](https://github.com/apache/arrow-rs/pull/9816) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- \[Json\] Remove arrow-data dependency from arrow-json [\#9812](https://github.com/apache/arrow-rs/pull/9812) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) -- Replace `BooleanBufferBuilder` with `NullBufferBuilder` in arrow-json if applicable [\#9811](https://github.com/apache/arrow-rs/pull/9811) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) -- refactor\(ipc\): derive Default for CompressionContext [\#9809](https://github.com/apache/arrow-rs/pull/9809) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mbutrovich](https://github.com/mbutrovich)) -- fix\(ipc\): reader misalignment when skipping ListView / LargeListView columns [\#9806](https://github.com/apache/arrow-rs/pull/9806) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pchintar](https://github.com/pchintar)) -- fix\(ipc\): Avoid panic on malformed compressed buffer prefix [\#9802](https://github.com/apache/arrow-rs/pull/9802) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pchintar](https://github.com/pchintar)) -- parquet: fix panic in DeltaByteArrayDecoder on invalid prefix lengths [\#9797](https://github.com/apache/arrow-rs/pull/9797) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([pchintar](https://github.com/pchintar)) -- feat\(parquet\): fuse level encoding with counting and histogram updates [\#9795](https://github.com/apache/arrow-rs/pull/9795) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) -- Expose ColumnCloseResult on ArrowColumnChunk [\#9773](https://github.com/apache/arrow-rs/pull/9773) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([leoyvens](https://github.com/leoyvens)) -- feat: make FFI structs fields `pub` [\#9772](https://github.com/apache/arrow-rs/pull/9772) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([ashdnazg](https://github.com/ashdnazg)) -- chore: Refine the error message for List to non List cast [\#9757](https://github.com/apache/arrow-rs/pull/9757) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([comphead](https://github.com/comphead)) -- refactor\(parquet\): replace magic `8` literals with named constants [\#9751](https://github.com/apache/arrow-rs/pull/9751) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) -- feat\(ipc\): add with\_skip\_validation to StreamDecoder [\#9749](https://github.com/apache/arrow-rs/pull/9749) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pantShrey](https://github.com/pantShrey)) -- remove panics in unshred variant [\#9741](https://github.com/apache/arrow-rs/pull/9741) ([friendlymatthew](https://github.com/friendlymatthew)) -- Add benchmark for ListView interleave [\#9738](https://github.com/apache/arrow-rs/pull/9738) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([vegarsti](https://github.com/vegarsti)) -- arrow-arith: fix 'occured' -\> 'occurred' in arity.rs comments [\#9736](https://github.com/apache/arrow-rs/pull/9736) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([SAY-5](https://github.com/SAY-5)) -- Refactor `RleEncoder::flush_bit_packed_run` to make flow clearer [\#9735](https://github.com/apache/arrow-rs/pull/9735) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://github.com/etseidl)) -- Fix RecordBatch::normalize\(\) null bitmap bug and add StructArray::flatten\(\) [\#9733](https://github.com/apache/arrow-rs/pull/9733) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([sqd](https://github.com/sqd)) -- Add benchmark for cast from/to decimals [\#9729](https://github.com/apache/arrow-rs/pull/9729) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([klion26](https://github.com/klion26)) -- refactor\(arrow-avro\): use `Decoder::flush_block` in async reader [\#9726](https://github.com/apache/arrow-rs/pull/9726) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mzabaluev](https://github.com/mzabaluev)) -- fix: ParquetError when reading corrupt parquet file with truncated data instead of Panic [\#9725](https://github.com/apache/arrow-rs/pull/9725) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([xuzifu666](https://github.com/xuzifu666)) -- feat\(parquet\): add wide-schema writer overhead benchmark [\#9723](https://github.com/apache/arrow-rs/pull/9723) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) -- fix: correct accounting in `DictEncoder::estimated_memory_size`, `Interner::estimated_memory_size` [\#9720](https://github.com/apache/arrow-rs/pull/9720) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([mzabaluev](https://github.com/mzabaluev)) -- arrow-ipc: Write 0 offset buffer for length-0 variable-size arrays [\#9717](https://github.com/apache/arrow-rs/pull/9717) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([atwam](https://github.com/atwam)) -- \[Json\] Support `FixedSizeList` in json decoder [\#9715](https://github.com/apache/arrow-rs/pull/9715) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) -- chore\(deps\): bump actions/upload-pages-artifact from 4 to 5 [\#9713](https://github.com/apache/arrow-rs/pull/9713) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Fix clippy warning in fixed\_size\_binary\_array.rs [\#9712](https://github.com/apache/arrow-rs/pull/9712) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([AdamGS](https://github.com/AdamGS)) -- feat: add `has_non_empty_nulls` helper function in `OffsetBuffer` [\#9711](https://github.com/apache/arrow-rs/pull/9711) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) -- chore\(deps\): bump pytest from 7.2.0 to 9.0.3 in /parquet/pytest [\#9706](https://github.com/apache/arrow-rs/pull/9706) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([dependabot[bot]](https://github.com/apps/dependabot)) -- Fedora license audit [\#9704](https://github.com/apache/arrow-rs/pull/9704) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([michel-slm](https://github.com/michel-slm)) -- \[Variant\] Take top-level nulls into consideration when extracting perfectly shredded children [\#9702](https://github.com/apache/arrow-rs/pull/9702) ([AdamGS](https://github.com/AdamGS)) -- feat\(parquet\): add `push_decoder` benchmark for `PushBuffers` overhead [\#9696](https://github.com/apache/arrow-rs/pull/9696) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) -- Add mutable bitwise operations to `BooleanArray` and `NullBuffer::union_many` [\#9692](https://github.com/apache/arrow-rs/pull/9692) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mbutrovich](https://github.com/mbutrovich)) -- chore\(deps\): update hashbrown requirement from 0.16.0 to 0.17.0 [\#9691](https://github.com/apache/arrow-rs/pull/9691) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([dependabot[bot]](https://github.com/apps/dependabot)) -- chore\(deps\): bump actions/github-script from 8 to 9 [\#9690](https://github.com/apache/arrow-rs/pull/9690) ([dependabot[bot]](https://github.com/apps/dependabot)) -- minor: Re-enable CDC bench [\#9686](https://github.com/apache/arrow-rs/pull/9686) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://github.com/etseidl)) -- \[Variant\] Add `VariantArrayBuilder::append_nulls` API [\#9685](https://github.com/apache/arrow-rs/pull/9685) ([sdf-jkl](https://github.com/sdf-jkl)) -- feat\(parquet\): add struct-column writer benchmarks [\#9679](https://github.com/apache/arrow-rs/pull/9679) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) -- \[Arrow\] Add API to check if `Field` has a valid `ExtensionType` [\#9677](https://github.com/apache/arrow-rs/pull/9677) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([sdf-jkl](https://github.com/sdf-jkl)) -- \[Variant\] `variant_get` should follow JSONPath semantics for Field path element [\#9676](https://github.com/apache/arrow-rs/pull/9676) ([sdf-jkl](https://github.com/sdf-jkl)) -- ParquetMetaDataPushDecoder API to clear all buffered ranges [\#9673](https://github.com/apache/arrow-rs/pull/9673) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([nathanb9](https://github.com/nathanb9)) -- Fix union cast incorrectness for duplicate field names [\#9666](https://github.com/apache/arrow-rs/pull/9666) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([friendlymatthew](https://github.com/friendlymatthew)) -- chore: re-export `MAX_INLINE_VIEW_LEN` from `arrow_data` [\#9665](https://github.com/apache/arrow-rs/pull/9665) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) -- No longer allow BIT\_PACKED level encoding in Parquet writer [\#9656](https://github.com/apache/arrow-rs/pull/9656) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://github.com/etseidl)) -- feat\(parquet\): add sparse-column writer benchmarks [\#9654](https://github.com/apache/arrow-rs/pull/9654) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) -- Support `GenericListViewArray::new_unchecked` and refactor `ListView` json decoder [\#9648](https://github.com/apache/arrow-rs/pull/9648) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) -- \[Json\] Add json reader benchmarks for ListView [\#9647](https://github.com/apache/arrow-rs/pull/9647) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) -- fix\(parquet\): fix CDC panic on nested ListArrays with null entries [\#9644](https://github.com/apache/arrow-rs/pull/9644) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([kszucs](https://github.com/kszucs)) -- Add a test for reading nested REE data in json [\#9634](https://github.com/apache/arrow-rs/pull/9634) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- \[Variant\] Fix `variant_get` to return `List` instead of `List` [\#9631](https://github.com/apache/arrow-rs/pull/9631) ([liamzwbao](https://github.com/liamzwbao)) -- ci: use ubuntu-slim runner for lightweight CI jobs [\#9630](https://github.com/apache/arrow-rs/pull/9630) ([CuteChuanChuan](https://github.com/CuteChuanChuan)) -- Add bloom filter folding to automatically size SBBF filters [\#9628](https://github.com/apache/arrow-rs/pull/9628) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([adriangb](https://github.com/adriangb)) -- Add List and ListView take benchmarks [\#9626](https://github.com/apache/arrow-rs/pull/9626) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([AdamGS](https://github.com/AdamGS)) -- ParquetPushDecoder API to clear all buffered ranges [\#9624](https://github.com/apache/arrow-rs/pull/9624) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([nathanb9](https://github.com/nathanb9)) -- fix: handle missing dictionary batch for null-only columns in IPC reader [\#9623](https://github.com/apache/arrow-rs/pull/9623) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([joaquinhuigomez](https://github.com/joaquinhuigomez)) -- Fix `MutableBuffer::clear` [\#9622](https://github.com/apache/arrow-rs/pull/9622) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Rafferty97](https://github.com/Rafferty97)) -- feat\[arrow-ord\]: suppport REE comparisons [\#9621](https://github.com/apache/arrow-rs/pull/9621) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([asubiotto](https://github.com/asubiotto)) -- chore\(deps\): update sha2 requirement from 0.10 to 0.11 [\#9618](https://github.com/apache/arrow-rs/pull/9618) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([dependabot[bot]](https://github.com/apps/dependabot)) -- Expose option to set line terminator for CSV writer [\#9617](https://github.com/apache/arrow-rs/pull/9617) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([svranesevic](https://github.com/svranesevic)) -- \[Json\] Add json reader benchmarks for Map and REE [\#9616](https://github.com/apache/arrow-rs/pull/9616) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) -- deps: fix `object_store` breakage for 0.13.2 [\#9612](https://github.com/apache/arrow-rs/pull/9612) ([mzabaluev-flarion](https://github.com/mzabaluev-flarion)) -- \[Variant\] Support Binary/LargeBinary children [\#9610](https://github.com/apache/arrow-rs/pull/9610) ([AdamGS](https://github.com/AdamGS)) -- fix: use writer types in Skipper for resolved named record types [\#9605](https://github.com/apache/arrow-rs/pull/9605) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([ariel-miculas](https://github.com/ariel-miculas)) -- feat\(parquet\): derive `PartialEq` and `Eq` for `CdcOptions` [\#9602](https://github.com/apache/arrow-rs/pull/9602) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([kszucs](https://github.com/kszucs)) -- Add `finish_preserve_values` to `ArrayBuilder` trait [\#9601](https://github.com/apache/arrow-rs/pull/9601) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([adamreichold](https://github.com/adamreichold)) -- \[Variant\] extend shredded null handling for arrays [\#9599](https://github.com/apache/arrow-rs/pull/9599) ([sdf-jkl](https://github.com/sdf-jkl)) -- \[Variant\] Add unshredded `Struct` fast-path for `variant_get(..., Struct)` [\#9597](https://github.com/apache/arrow-rs/pull/9597) ([sdf-jkl](https://github.com/sdf-jkl)) -- Pre-reserve output capacity in ByteView/ByteArray dictionary decoding [\#9590](https://github.com/apache/arrow-rs/pull/9590) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) -- \[Variant\] Align cast logic for variant\_get to cast kernel for numeric/bool types [\#9563](https://github.com/apache/arrow-rs/pull/9563) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([klion26](https://github.com/klion26)) -- Add support to cast from `UnionArray` [\#9544](https://github.com/apache/arrow-rs/pull/9544) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([friendlymatthew](https://github.com/friendlymatthew)) -- Support `ListView` codec in arrow-json [\#9503](https://github.com/apache/arrow-rs/pull/9503) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- impl `FromStr` for `DatePart` [\#9931](https://github.com/apache/arrow-rs/pull/9931) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([sdf-jkl](https://github.com/sdf-jkl)) +- Pre-size dedup HashTable in GenericByteDictionaryBuilder::with\_capacity [\#9908](https://github.com/apache/arrow-rs/pull/9908) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rabenhorst](https://github.com/rabenhorst)) +- \[arrow-array\] Use consistent `value_length` name in FixedSizeBinaryArray [\#9905](https://github.com/apache/arrow-rs/pull/9905) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- replace Dictionary::try\_new\(\) calls with with\_values. [\#9894](https://github.com/apache/arrow-rs/pull/9894) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Rich-T-kid](https://github.com/Rich-T-kid)) +- API to help with the pattern of 'replaces the values of the REE array [\#9891](https://github.com/apache/arrow-rs/pull/9891) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Rich-T-kid](https://github.com/Rich-T-kid)) +- fix\(arrow-csv\): bound RecordDecoder::flush offset accumulation [\#9886](https://github.com/apache/arrow-rs/pull/9886) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([masumi-ryugo](https://github.com/masumi-ryugo)) +- fix\(parquet\): bound schema num\_children before Vec::with\_capacity [\#9884](https://github.com/apache/arrow-rs/pull/9884) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([masumi-ryugo](https://github.com/masumi-ryugo)) +- feat\(arrow-string\): concat\_elements for view, fixed binary [\#9876](https://github.com/apache/arrow-rs/pull/9876) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([theirix](https://github.com/theirix)) +- Prevent `FixedSizeBinaryArray` `i32` offset overflows \(try 2\) [\#9872](https://github.com/apache/arrow-rs/pull/9872) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- \[arrow-string\]: add `like::eq_ascii_ignore_case` kernel [\#9871](https://github.com/apache/arrow-rs/pull/9871) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([albertlockett](https://github.com/albertlockett)) +- fix\(parquet\): Prevent negative list sizes in Thrift compact protocol parser [\#9868](https://github.com/apache/arrow-rs/pull/9868) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([masumi-ryugo](https://github.com/masumi-ryugo)) +- \[PARQUET\] Allow `UNKNOWN` logical type annotation on any physical type [\#9855](https://github.com/apache/arrow-rs/pull/9855) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://github.com/etseidl)) +- \[arrow-ipc\]: dictionary builders for delta - doc fix and integration tests for nested types [\#9853](https://github.com/apache/arrow-rs/pull/9853) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([albertlockett](https://github.com/albertlockett)) +- fix\(arrow-cast\): fix incorrect conversion [\#9852](https://github.com/apache/arrow-rs/pull/9852) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([bboissin](https://github.com/bboissin)) +- chore\[benches\]: add REE interleave benchmarks [\#9849](https://github.com/apache/arrow-rs/pull/9849) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([asubiotto](https://github.com/asubiotto)) +- test\(parquet\): replace `InMemoryArrayReader` with `PrimitiveArrayReader` in tests [\#9847](https://github.com/apache/arrow-rs/pull/9847) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- REE row conversion speed up [\#9845](https://github.com/apache/arrow-rs/pull/9845) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Rich-T-kid](https://github.com/Rich-T-kid)) +- fix\(parquet\): Avoid panic on malformed thrift bool fields in parquet metadata [\#9840](https://github.com/apache/arrow-rs/pull/9840) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([BoazC-MSFT](https://github.com/BoazC-MSFT)) +- fix\(parquet\): avoid panic on ColumnIndex length mismatch [\#9833](https://github.com/apache/arrow-rs/pull/9833) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([pchintar](https://github.com/pchintar)) +- configurable data page v2 compression threshold [\#9826](https://github.com/apache/arrow-rs/pull/9826) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([leoyvens](https://github.com/leoyvens)) +- Prevent `ArrayData::slice` length overflow [\#9813](https://github.com/apache/arrow-rs/pull/9813) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Fix JSON reader panic for non-nullable zero-size FixedSizeList [\#9810](https://github.com/apache/arrow-rs/pull/9810) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- \[Parquet\] Do not panic when trying to skip records in delta encoded files using non-standard block sizes [\#9794](https://github.com/apache/arrow-rs/pull/9794) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://github.com/etseidl)) diff --git a/Cargo.toml b/Cargo.toml index 52140a05287c..9a13149626e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,7 +68,7 @@ exclude = [ ] [workspace.package] -version = "58.2.0" +version = "58.3.0" homepage = "https://github.com/apache/arrow-rs" repository = "https://github.com/apache/arrow-rs" authors = ["Apache Arrow "] @@ -85,26 +85,26 @@ edition = "2024" rust-version = "1.85" [workspace.dependencies] -arrow = { version = "58.2.0", path = "./arrow", default-features = false } -arrow-arith = { version = "58.2.0", path = "./arrow-arith" } -arrow-array = { version = "58.2.0", path = "./arrow-array" } -arrow-buffer = { version = "58.2.0", path = "./arrow-buffer" } -arrow-cast = { version = "58.2.0", path = "./arrow-cast" } -arrow-csv = { version = "58.2.0", path = "./arrow-csv" } -arrow-data = { version = "58.2.0", path = "./arrow-data" } -arrow-ipc = { version = "58.2.0", path = "./arrow-ipc" } -arrow-json = { version = "58.2.0", path = "./arrow-json" } -arrow-ord = { version = "58.2.0", path = "./arrow-ord" } -arrow-pyarrow = { version = "58.2.0", path = "./arrow-pyarrow" } -arrow-row = { version = "58.2.0", path = "./arrow-row" } -arrow-schema = { version = "58.2.0", path = "./arrow-schema" } -arrow-select = { version = "58.2.0", path = "./arrow-select" } -arrow-string = { version = "58.2.0", path = "./arrow-string" } -parquet = { version = "58.2.0", path = "./parquet", default-features = false } -parquet-geospatial = { version = "58.2.0", path = "./parquet-geospatial" } -parquet-variant = { version = "58.2.0", path = "./parquet-variant" } -parquet-variant-json = { version = "58.2.0", path = "./parquet-variant-json" } -parquet-variant-compute = { version = "58.2.0", path = "./parquet-variant-compute" } +arrow = { version = "58.3.0", path = "./arrow", default-features = false } +arrow-arith = { version = "58.3.0", path = "./arrow-arith" } +arrow-array = { version = "58.3.0", path = "./arrow-array" } +arrow-buffer = { version = "58.3.0", path = "./arrow-buffer" } +arrow-cast = { version = "58.3.0", path = "./arrow-cast" } +arrow-csv = { version = "58.3.0", path = "./arrow-csv" } +arrow-data = { version = "58.3.0", path = "./arrow-data" } +arrow-ipc = { version = "58.3.0", path = "./arrow-ipc" } +arrow-json = { version = "58.3.0", path = "./arrow-json" } +arrow-ord = { version = "58.3.0", path = "./arrow-ord" } +arrow-pyarrow = { version = "58.3.0", path = "./arrow-pyarrow" } +arrow-row = { version = "58.3.0", path = "./arrow-row" } +arrow-schema = { version = "58.3.0", path = "./arrow-schema" } +arrow-select = { version = "58.3.0", path = "./arrow-select" } +arrow-string = { version = "58.3.0", path = "./arrow-string" } +parquet = { version = "58.3.0", path = "./parquet", default-features = false } +parquet-geospatial = { version = "58.3.0", path = "./parquet-geospatial" } +parquet-variant = { version = "58.3.0", path = "./parquet-variant" } +parquet-variant-json = { version = "58.3.0", path = "./parquet-variant-json" } +parquet-variant-compute = { version = "58.3.0", path = "./parquet-variant-compute" } chrono = { version = "0.4.40", default-features = false, features = ["clock"] } diff --git a/dev/release/update_change_log.sh b/dev/release/update_change_log.sh index d4c8fa250141..21b4c2d2956d 100755 --- a/dev/release/update_change_log.sh +++ b/dev/release/update_change_log.sh @@ -29,8 +29,8 @@ set -e -SINCE_TAG="58.1.0" -FUTURE_RELEASE="58.2.0" +SINCE_TAG="58.2.0" +FUTURE_RELEASE="58.3.0" SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" From c2021f1c4a15c4b420cb78f1bc9dc371d6e30ccc Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 7 May 2026 10:48:21 -0400 Subject: [PATCH 19/33] Fix MSRV check by checking in Cargo.lock (#9941) NOTE: All of this PR is `Cargo.lock`. I swear it is easy to review... # Which issue does this PR close? - Closes #9938. - Modeled on #9902. # Rationale for this change The guidance on lock files from the Cargo folks changed a while ago: https://blog.rust-lang.org/2023/08/29/committing-lockfiles/ The MSRV check is failing on `main` because dependency resolution currently uses the latest compatible versions from crates.io for each package. The newest `tonic` release now require a newer Rust version than the workspace MSRV. Here is the reported CI failure: https://github.com/apache/arrow-rs/actions/runs/25472344356/job/74738606768 ```text error: rustc 1.85.0 is not supported by the following packages: tonic@0.14.6 requires rustc 1.88 tonic-prost@0.14.6 requires rustc 1.88 ``` # What changes are included in this PR? This PR checks in a root `Cargo.lock` so CI verifies MSRV against the dependency set we control, rather than the tip of all dependency ranges. The generated lockfile pins the `tonic` 0.14 crates to `0.14.5`, which supports Rust 1.85. Note this does not change code in the crates. It only pins dependency resolution for workspace builds and CI. This will result in more dependabot PRs to explicitly update the crate versions, but I think that is a good thing. I think the existing config file will work fine https://github.com/apache/arrow-rs/blob/main/.github/dependabot.yml # Are these changes tested? Yes, by CI This passed locally with Rust 1.85.0. # Are there any user-facing changes? No. This only checks in the root lockfile used for dependency resolution in workspace builds and CI. --- Cargo.lock | 4598 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 4598 insertions(+) create mode 100644 Cargo.lock diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 000000000000..a3465ba9e458 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,4598 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.60.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.60.2", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "apache-avro" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" +dependencies = [ + "bigdecimal", + "bon", + "digest 0.10.7", + "log", + "miniz_oxide", + "num-bigint", + "quad-rand", + "rand 0.9.4", + "regex-lite", + "serde", + "serde_bytes", + "serde_json", + "strum", + "strum_macros 0.27.2", + "thiserror 2.0.18", + "uuid", +] + +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "arrow" +version = "58.2.0" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-pyarrow", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", + "bytes", + "chrono", + "criterion", + "half", + "memmap2", + "rand 0.9.4", + "serde", +] + +[[package]] +name = "arrow-arith" +version = "58.2.0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "58.2.0" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "criterion", + "futures", + "half", + "hashbrown 0.17.0", + "num-complex", + "num-integer", + "num-traits", + "rand 0.9.4", +] + +[[package]] +name = "arrow-avro" +version = "58.2.0" +dependencies = [ + "apache-avro", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "async-stream", + "bytes", + "bzip2", + "crc", + "criterion", + "flate2", + "futures", + "half", + "indexmap", + "liblzma", + "md5", + "num-bigint", + "object_store", + "once_cell", + "rand 0.9.4", + "serde", + "serde_json", + "sha2", + "snap", + "strum_macros 0.28.0", + "tempfile", + "tokio", + "uuid", + "zstd", +] + +[[package]] +name = "arrow-buffer" +version = "58.2.0" +dependencies = [ + "bytes", + "criterion", + "half", + "num-bigint", + "num-traits", + "rand 0.9.4", +] + +[[package]] +name = "arrow-cast" +version = "58.2.0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "comfy-table", + "criterion", + "half", + "insta", + "lexical-core", + "num-traits", + "rand 0.9.4", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "58.2.0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-schema", + "bytes", + "chrono", + "csv", + "csv-core", + "futures", + "regex", + "tempfile", + "tokio", +] + +[[package]] +name = "arrow-data" +version = "58.2.0" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-flight" +version = "58.2.0" +dependencies = [ + "anyhow", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", + "assert_cmd", + "base64", + "bytes", + "clap", + "futures", + "http", + "http-body", + "hyper-util", + "once_cell", + "paste", + "pin-project-lite", + "prost", + "prost-types", + "tempfile", + "tokio", + "tokio-stream", + "tonic", + "tonic-prost", + "tower", + "tracing-log", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "arrow-integration-test" +version = "58.2.0" +dependencies = [ + "arrow", + "arrow-buffer", + "hex", + "num-bigint", + "num-traits", + "serde", + "serde_json", +] + +[[package]] +name = "arrow-integration-testing" +version = "58.2.0" +dependencies = [ + "arrow", + "arrow-buffer", + "arrow-flight", + "arrow-integration-test", + "clap", + "flate2", + "futures", + "prost", + "serde_json", + "tempfile", + "tokio", + "tonic", + "tracing-subscriber", +] + +[[package]] +name = "arrow-ipc" +version = "58.2.0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "bytes", + "criterion", + "flatbuffers", + "lz4_flex", + "memmap2", + "tempfile", + "tokio", + "zstd", +] + +[[package]] +name = "arrow-json" +version = "58.2.0" +dependencies = [ + "arbitrary", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "bytes", + "chrono", + "criterion", + "flate2", + "futures", + "half", + "indexmap", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "rand 0.9.4", + "ryu", + "serde", + "serde_core", + "serde_json", + "simdutf8", + "tokio", +] + +[[package]] +name = "arrow-ord" +version = "58.2.0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "half", + "rand 0.9.4", +] + +[[package]] +name = "arrow-pyarrow" +version = "58.2.0" +dependencies = [ + "arrow-array", + "arrow-data", + "arrow-schema", + "pyo3", +] + +[[package]] +name = "arrow-row" +version = "58.2.0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "half", + "rand 0.9.4", +] + +[[package]] +name = "arrow-schema" +version = "58.2.0" +dependencies = [ + "bitflags", + "criterion", + "insta", + "postcard", + "serde", + "serde_core", + "serde_json", +] + +[[package]] +name = "arrow-select" +version = "58.2.0" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", + "rand 0.9.4", +] + +[[package]] +name = "arrow-string" +version = "58.2.0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "assert_cmd" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39bae1d3fa576f7c6519514180a72559268dd7d1fe104070956cb687bc6673bd" +dependencies = [ + "anstyle", + "bstr", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "aws-lc-rs" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bigdecimal" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bon" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "brotli" +version = "8.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.2.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "num-traits", + "serde", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", + "terminal_size", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "console" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" +dependencies = [ + "encode_unicode", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "futures", + "itertools 0.13.0", + "num-traits", + "oorandom", + "page_size", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools 0.13.0", +] + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.0", + "const-oid", + "crypto-common 0.2.1", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "gen" +version = "0.1.0" +dependencies = [ + "prost-build", + "tonic-prost-build", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "geo-traits" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7c353d12a704ccfab1ba8bfb1a7fe6cb18b665bf89d37f4f7890edcd260206" +dependencies = [ + "geo-types", +] + +[[package]] +name = "geo-types" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94776032c45f950d30a13af6113c2ad5625316c9abfbccee4dd5a6695f8fe0f5" +dependencies = [ + "approx", + "num-traits", + "serde", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasip2", + "wasip3", +] + +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "rand 0.9.4", + "rand_distr", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + +[[package]] +name = "hybrid-array" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d46837a0ed51fe95bd3b05de33cd64a1ee88fc797477ca48446872504507c5" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.0", + "serde", + "serde_core", +] + +[[package]] +name = "insta" +version = "1.47.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4a6248eb93a4401ed2f37dfe8ea592d3cf05b7cf4f8efa867b6895af7e094e" +dependencies = [ + "console", + "once_cell", + "similar", + "tempfile", +] + +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libbz2-rs-sys" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3a6a8c165077efc8f3a971534c50ea6a1a18b329ef4a66e897a7e3a1494565f" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "liblzma" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6033b77c21d1f56deeae8014eb9fbe7bdf1765185a6c508b5ca82eeaed7f899" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a60851d15cd8c5346eca4ab8babff585be2ae4bc8097c067291d3ffe2add3b6" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "lz4_flex" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md5" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.60.2", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "object_store" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +dependencies = [ + "async-trait", + "base64", + "bytes", + "chrono", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body-util", + "httparse", + "humantime", + "hyper", + "itertools 0.14.0", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.10.1", + "reqwest", + "ring", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "parquet" +version = "58.2.0" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64", + "brotli", + "bytes", + "chrono", + "clap", + "crc32fast", + "criterion", + "flate2", + "futures", + "half", + "hashbrown 0.17.0", + "insta", + "lz4_flex", + "num-bigint", + "num-integer", + "num-traits", + "object_store", + "parquet-geospatial", + "parquet-variant", + "parquet-variant-compute", + "parquet-variant-json", + "paste", + "rand 0.9.4", + "ring", + "seq-macro", + "serde", + "serde_json", + "simdutf8", + "snap", + "sysinfo", + "tempfile", + "thrift", + "tokio", + "twox-hash", + "zstd", +] + +[[package]] +name = "parquet-geospatial" +version = "58.2.0" +dependencies = [ + "arrow-schema", + "geo-traits", + "serde", + "serde_json", + "wkb", + "wkt", +] + +[[package]] +name = "parquet-variant" +version = "58.2.0" +dependencies = [ + "arrow", + "arrow-schema", + "chrono", + "criterion", + "half", + "indexmap", + "num-traits", + "paste", + "rand 0.9.4", + "simdutf8", + "uuid", +] + +[[package]] +name = "parquet-variant-compute" +version = "58.2.0" +dependencies = [ + "arrow", + "arrow-schema", + "chrono", + "criterion", + "half", + "indexmap", + "parquet-variant", + "parquet-variant-json", + "rand 0.9.4", + "serde_json", + "uuid", +] + +[[package]] +name = "parquet-variant-json" +version = "58.2.0" +dependencies = [ + "arrow-schema", + "base64", + "chrono", + "parquet-variant", + "serde_json", + "uuid", +] + +[[package]] +name = "parquet_derive" +version = "58.2.0" +dependencies = [ + "parquet", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "parquet_derive_test" +version = "58.2.0" +dependencies = [ + "chrono", + "parquet", + "parquet_derive", + "uuid", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf0d9e68100b3a7989b4901972f265cd542e560a3a8a724e1e20322f4d06ce9" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a990e22f43e84855daf260dded30524ef4a9021cc7541c26540500a50b624389" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "serde", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "difflib", + "predicates-core", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + +[[package]] +name = "pyo3" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", +] + +[[package]] +name = "pyo3-build-config" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "quad-rand" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" + +[[package]] +name = "quick-xml" +version = "0.39.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "721da970c312655cde9b4ffe0547f20a8494866a4af5ff51f18b7c633d0c870b" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.4", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "aws-lc-rs", + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sysinfo" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix", + "windows-sys 0.60.2", +] + +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "110a78583f19d5cdb2c5ccf321d1290344e71313c6c37d43520d386027d18386" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.11+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "tonic" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" +dependencies = [ + "async-trait", + "axum", + "base64", + "bytes", + "flate2", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "rustls-native-certs", + "socket2", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", + "webpki-roots", + "zstd", +] + +[[package]] +name = "tonic-build" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1882ac3bf5ef12877d7ed57aad87e75154c11931c2ba7e6cde5e22d63522c734" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tonic-prost" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3144df636917574672e93d0f56d7edec49f90305749c668df5101751bb8f95a" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn", + "tempfile", + "tonic-build", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "thread_local", + "tracing", + "tracing-core", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + +[[package]] +name = "typenum" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winnow" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "wkb" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a120b336c7ad17749026d50427c23d838ecb50cd64aaea6254b5030152f890a9" +dependencies = [ + "byteorder", + "geo-traits", + "num_enum", + "thiserror 1.0.69", +] + +[[package]] +name = "wkt" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efb2b923ccc882312e559ffaa832a055ba9d1ac0cc8e86b3e25453247e4b81d7" +dependencies = [ + "geo-traits", + "geo-types", + "log", + "num-traits", + "thiserror 1.0.69", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zlib-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] From 13f5f940645fd17527a8397b296f04dc1f7da1d7 Mon Sep 17 00:00:00 2001 From: Hippolyte Barraud Date: Thu, 7 May 2026 13:16:51 -0400 Subject: [PATCH 20/33] feat(parquet): compact level representation with generic writer dispatch (#9831) # Which issue does this PR close? - Spawn off from #9653 - Contributes to #9731 # Rationale for this change See #9731 # What changes are included in this PR? Represent definition and repetition levels as `LevelData`/`LevelDataRef` with `Absent`, `Materialized`, and `Uniform` variants, and thread this through Arrow level generation, CDC chunking, and the generic column writer. Uniform level runs, such as required fields and all-null pages, can now be encoded without materializing dense `Vec` buffers. Add bulk run support to `LevelEncoder`/`RleEncoder` so repeated levels are encoded in amortized O(1) after the RLE warmup, while preserving histogram, row count, null count, page splitting, and CDC chunk accounting. # Are these changes tested? All tests passing. Coverage exercises bulk RLE level encoding, compact/uniform `LevelData` slicing and writer roundtrips across Parquet v1/v2, and CDC/Arrow writer behavior including all-null and nested-level cases. # Are there any user-facing changes? None. --------- Signed-off-by: Hippolyte Barraud Co-authored-by: Andrew Lamb --- parquet/src/arrow/arrow_writer/levels.rs | 417 +++++++++++++-------- parquet/src/arrow/arrow_writer/mod.rs | 34 +- parquet/src/column/chunker/cdc.rs | 104 ++++-- parquet/src/column/writer/mod.rs | 450 ++++++++++++++++++++--- parquet/src/encodings/levels.rs | 144 ++++++++ 5 files changed, 905 insertions(+), 244 deletions(-) diff --git a/parquet/src/arrow/arrow_writer/levels.rs b/parquet/src/arrow/arrow_writer/levels.rs index d23873278ed3..1f66de349df0 100644 --- a/parquet/src/arrow/arrow_writer/levels.rs +++ b/parquet/src/arrow/arrow_writer/levels.rs @@ -41,6 +41,7 @@ //! \[1\] [parquet-format#nested-encoding](https://github.com/apache/parquet-format#nested-encoding) use crate::column::chunker::CdcChunk; +use crate::column::writer::LevelDataRef; use crate::errors::{ParquetError, Result}; use arrow_array::cast::AsArray; use arrow_array::{Array, ArrayRef, OffsetSizeTrait}; @@ -316,7 +317,7 @@ impl LevelInfoBuilder { |child: &mut LevelInfoBuilder, start_idx: usize, end_idx: usize| { child.write(start_idx..end_idx); child.visit_leaves(|leaf| { - let rep_levels = leaf.rep_levels.as_mut().unwrap(); + let rep_levels = leaf.rep_levels.materialize_mut().unwrap(); let mut rev = rep_levels.iter_mut().rev(); let mut remaining = end_idx - start_idx; @@ -336,17 +337,18 @@ impl LevelInfoBuilder { }) }; + // In a list column, each row falls into one of three categories: + // - "null": the list slot is absent (!is_valid), encoded at def_level - 2 + // - "empty": the list slot is present but has zero elements + // (offsets[i] == offsets[i+1]), encoded at def_level - 1 + // - non-empty: the list slot has child values, which are recursed into + // + // Consecutive runs of null or empty rows are batched and written together. let write_null_run = |child: &mut LevelInfoBuilder, count: usize| { if count > 0 { child.visit_leaves(|leaf| { - leaf.rep_levels - .as_mut() - .unwrap() - .extend(std::iter::repeat_n(ctx.rep_level - 1, count)); - leaf.def_levels - .as_mut() - .unwrap() - .extend(std::iter::repeat_n(ctx.def_level - 2, count)); + leaf.append_rep_level_run(ctx.rep_level - 1, count); + leaf.append_def_level_run(ctx.def_level - 2, count); }); } }; @@ -354,14 +356,8 @@ impl LevelInfoBuilder { let write_empty_run = |child: &mut LevelInfoBuilder, count: usize| { if count > 0 { child.visit_leaves(|leaf| { - leaf.rep_levels - .as_mut() - .unwrap() - .extend(std::iter::repeat_n(ctx.rep_level - 1, count)); - leaf.def_levels - .as_mut() - .unwrap() - .extend(std::iter::repeat_n(ctx.def_level - 1, count)); + leaf.append_rep_level_run(ctx.rep_level - 1, count); + leaf.append_def_level_run(ctx.def_level - 1, count); }); } }; @@ -431,7 +427,7 @@ impl LevelInfoBuilder { |child: &mut LevelInfoBuilder, start_idx: usize, end_idx: usize| { child.write(start_idx..end_idx); child.visit_leaves(|leaf| { - let rep_levels = leaf.rep_levels.as_mut().unwrap(); + let rep_levels = leaf.rep_levels.materialize_mut().unwrap(); let mut rev = rep_levels.iter_mut().rev(); let mut remaining = end_idx - start_idx; @@ -453,19 +449,15 @@ impl LevelInfoBuilder { let write_empty_slice = |child: &mut LevelInfoBuilder| { child.visit_leaves(|leaf| { - let rep_levels = leaf.rep_levels.as_mut().unwrap(); - rep_levels.push(ctx.rep_level - 1); - let def_levels = leaf.def_levels.as_mut().unwrap(); - def_levels.push(ctx.def_level - 1); + leaf.append_rep_level_run(ctx.rep_level - 1, 1); + leaf.append_def_level_run(ctx.def_level - 1, 1); }) }; let write_null_slice = |child: &mut LevelInfoBuilder| { child.visit_leaves(|leaf| { - let rep_levels = leaf.rep_levels.as_mut().unwrap(); - rep_levels.push(ctx.rep_level - 1); - let def_levels = leaf.def_levels.as_mut().unwrap(); - def_levels.push(ctx.def_level - 2); + leaf.append_rep_level_run(ctx.rep_level - 1, 1); + leaf.append_def_level_run(ctx.def_level - 2, 1); }) }; @@ -513,13 +505,8 @@ impl LevelInfoBuilder { for child in children { child.visit_leaves(|info| { let len = range.end - range.start; - - let def_levels = info.def_levels.as_mut().unwrap(); - def_levels.extend(std::iter::repeat_n(ctx.def_level - 1, len)); - - if let Some(rep_levels) = info.rep_levels.as_mut() { - rep_levels.extend(std::iter::repeat_n(ctx.rep_level, len)); - } + info.append_def_level_run(ctx.def_level - 1, len); + info.append_rep_level_run(ctx.rep_level, len); }) } }; @@ -579,7 +566,7 @@ impl LevelInfoBuilder { child.write(values_start..values_end); child.visit_leaves(|leaf| { - let rep_levels = leaf.rep_levels.as_mut().unwrap(); + let rep_levels = leaf.rep_levels.materialize_mut().unwrap(); let row_indices = (0..fixed_size) .rev() @@ -605,10 +592,8 @@ impl LevelInfoBuilder { let write_empty = |child: &mut LevelInfoBuilder, start_idx: usize, end_idx: usize| { let len = end_idx - start_idx; child.visit_leaves(|leaf| { - let rep_levels = leaf.rep_levels.as_mut().unwrap(); - rep_levels.extend(std::iter::repeat_n(ctx.rep_level - 1, len)); - let def_levels = leaf.def_levels.as_mut().unwrap(); - def_levels.extend(std::iter::repeat_n(ctx.def_level - 1, len)); + leaf.append_rep_level_run(ctx.rep_level - 1, len); + leaf.append_def_level_run(ctx.def_level - 1, len); }) }; @@ -634,10 +619,8 @@ impl LevelInfoBuilder { } // Add null row child.visit_leaves(|leaf| { - let rep_levels = leaf.rep_levels.as_mut().unwrap(); - rep_levels.push(ctx.rep_level - 1); - let def_levels = leaf.def_levels.as_mut().unwrap(); - def_levels.push(ctx.def_level - 2); + leaf.append_rep_level_run(ctx.rep_level - 1, 1); + leaf.append_def_level_run(ctx.def_level - 2, 1); }) } } @@ -655,37 +638,34 @@ impl LevelInfoBuilder { fn write_leaf(info: &mut ArrayLevels, range: Range) { let len = range.end - range.start; - match &mut info.def_levels { - Some(def_levels) => { - def_levels.reserve(len); - info.non_null_indices.reserve(len); - - match &info.logical_nulls { - Some(nulls) => { - assert!(range.end <= nulls.len()); - let nulls = nulls.inner(); - def_levels.extend(range.clone().map(|i| { - // Safety: range.end was asserted to be in bounds earlier - let valid = unsafe { nulls.value_unchecked(i) }; - info.max_def_level - (!valid as i16) - })); - info.non_null_indices.extend( - BitIndexIterator::new(nulls.inner(), nulls.offset() + range.start, len) - .map(|i| i + range.start), - ); - } - None => { - let iter = std::iter::repeat_n(info.max_def_level, len); - def_levels.extend(iter); - info.non_null_indices.extend(range); - } + if matches!(info.def_levels, LevelData::Absent) { + info.non_null_indices.extend(range.clone()); + } else { + let max_def_level = info.max_def_level; + match &info.logical_nulls { + Some(nulls) => { + assert!(range.end <= nulls.len()); + let nulls = nulls.inner(); + info.def_levels.extend_from_iter(range.clone().map(|i| { + // Safety: range.end was asserted to be in bounds earlier + let valid = unsafe { nulls.value_unchecked(i) }; + max_def_level - (!valid as i16) + })); + info.non_null_indices.reserve(len); + info.non_null_indices.extend( + BitIndexIterator::new(nulls.inner(), nulls.offset() + range.start, len) + .map(|i| i + range.start), + ); + } + None => { + info.append_def_level_run(max_def_level, len); + info.non_null_indices.extend(range.clone()); } } - None => info.non_null_indices.extend(range), } - if let Some(rep_levels) = &mut info.rep_levels { - rep_levels.extend(std::iter::repeat_n(info.max_rep_level, len)) + if !matches!(info.rep_levels, LevelData::Absent) { + info.append_rep_level_run(info.max_rep_level, len); } } @@ -754,17 +734,142 @@ impl LevelInfoBuilder { /// The data necessary to write a primitive Arrow array to parquet, taking into account /// any non-primitive parents it may have in the arrow representation +#[derive(Debug, Clone)] +pub(crate) enum LevelData { + Absent, + Materialized(Vec), + Uniform { value: i16, count: usize }, +} + +// Compare logical level contents rather than physical representation, so a +// uniform run compares equal to the equivalent materialized buffer. +impl PartialEq for LevelData { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Absent, Self::Absent) => true, + (Self::Materialized(a), Self::Materialized(b)) => a == b, + (Self::Uniform { value: v, count: n }, Self::Materialized(b)) + | (Self::Materialized(b), Self::Uniform { value: v, count: n }) => { + b.len() == *n && b.iter().all(|x| x == v) + } + ( + Self::Uniform { + value: v1, + count: n1, + }, + Self::Uniform { + value: v2, + count: n2, + }, + ) => v1 == v2 && n1 == n2, + _ => false, + } + } +} + +impl Eq for LevelData {} + +impl LevelData { + fn new(present: bool) -> Self { + match present { + true => Self::Materialized(Vec::new()), + false => Self::Absent, + } + } + + pub(crate) fn as_ref(&self) -> LevelDataRef<'_> { + match self { + Self::Absent => LevelDataRef::Absent, + Self::Materialized(values) => LevelDataRef::Materialized(values), + Self::Uniform { value, count } => LevelDataRef::Uniform { + value: *value, + count: *count, + }, + } + } + + pub(crate) fn slice(&self, offset: usize, len: usize) -> Self { + match self { + Self::Absent => Self::Absent, + Self::Materialized(values) => Self::Materialized(values[offset..offset + len].to_vec()), + Self::Uniform { value, .. } => Self::Uniform { + value: *value, + count: len, + }, + } + } + + fn append_run(&mut self, value: i16, count: usize) { + if count == 0 { + return; + } + + match self { + // No physical level stream exists for this schema. Higher-level + // traversal may still append implicit levels, so this remains a no-op. + Self::Absent => {} + // Start compact: the first appended run can be represented without + // allocating a level buffer. + Self::Materialized(values) if values.is_empty() => { + *self = Self::Uniform { value, count }; + } + // Already materialized, so preserve the buffer representation and append. + Self::Materialized(values) => values.extend(std::iter::repeat_n(value, count)), + // Preserve the compact representation while the appended run has + // the same value. + Self::Uniform { + value: uniform_value, + count: uniform_count, + } if *uniform_value == value => { + *uniform_count += count; + } + // A different value breaks the uniform representation. Materialize + // the existing run, then append the new run to the buffer. + Self::Uniform { .. } => { + let values = self.materialize_mut().unwrap(); + values.extend(std::iter::repeat_n(value, count)); + } + } + } + + fn extend_from_iter(&mut self, iter: I) + where + I: IntoIterator, + { + if let Some(values) = self.materialize_mut() { + values.extend(iter); + } + } + + /// Convert a uniform run into a materialized buffer if needed, then return + /// the mutable level buffer. Returns `None` when no physical level stream exists. + fn materialize_mut(&mut self) -> Option<&mut Vec> { + match self { + Self::Absent => None, + Self::Materialized(values) => Some(values), + Self::Uniform { value, count } => { + let values = vec![*value; *count]; + *self = Self::Materialized(values); + match self { + Self::Materialized(values) => Some(values), + _ => unreachable!(), + } + } + } + } +} + #[derive(Debug, Clone)] pub(crate) struct ArrayLevels { /// Array's definition levels /// /// Present if `max_def_level != 0` - def_levels: Option>, + def_levels: LevelData, /// Array's optional repetition levels /// /// Present if `max_rep_level != 0` - rep_levels: Option>, + rep_levels: LevelData, /// The corresponding array identifying non-null slices of data /// from the primitive array @@ -807,8 +912,8 @@ impl ArrayLevels { let logical_nulls = array.logical_nulls(); Self { - def_levels: (max_def_level != 0).then(Vec::new), - rep_levels: (max_rep_level != 0).then(Vec::new), + def_levels: LevelData::new(max_def_level != 0), + rep_levels: LevelData::new(max_rep_level != 0), non_null_indices: vec![], max_def_level, max_rep_level, @@ -821,12 +926,12 @@ impl ArrayLevels { &self.array } - pub fn def_levels(&self) -> Option<&[i16]> { - self.def_levels.as_deref() + pub(crate) fn def_level_data(&self) -> &LevelData { + &self.def_levels } - pub fn rep_levels(&self) -> Option<&[i16]> { - self.rep_levels.as_deref() + pub(crate) fn rep_level_data(&self) -> &LevelData { + &self.rep_levels } pub fn non_null_indices(&self) -> &[usize] { @@ -839,12 +944,8 @@ impl ArrayLevels { /// `non_null_indices`. The array is sliced to the range covered by /// those indices, and they are shifted to be relative to the slice. pub(crate) fn slice_for_chunk(&self, chunk: &CdcChunk) -> Self { - let def_levels = self.def_levels.as_ref().map(|levels| { - levels[chunk.level_offset..chunk.level_offset + chunk.num_levels].to_vec() - }); - let rep_levels = self.rep_levels.as_ref().map(|levels| { - levels[chunk.level_offset..chunk.level_offset + chunk.num_levels].to_vec() - }); + let def_levels = self.def_levels.slice(chunk.level_offset, chunk.num_levels); + let rep_levels = self.rep_levels.slice(chunk.level_offset, chunk.num_levels); // Select the non-null indices for this chunk. let nni = &self.non_null_indices[chunk.value_offset..chunk.value_offset + chunk.num_values]; @@ -870,6 +971,14 @@ impl ArrayLevels { logical_nulls, } } + + fn append_def_level_run(&mut self, value: i16, count: usize) { + self.def_levels.append_run(value, count); + } + + fn append_rep_level_run(&mut self, value: i16, count: usize) { + self.rep_levels.append_run(value, count); + } } #[cfg(test)] @@ -920,8 +1029,8 @@ mod tests { assert_eq!(levels.len(), 1); let expected = ArrayLevels { - def_levels: Some(vec![2; 10]), - rep_levels: Some(vec![0, 2, 2, 1, 2, 2, 2, 0, 1, 2]), + def_levels: LevelData::Materialized(vec![2; 10]), + rep_levels: LevelData::Materialized(vec![0, 2, 2, 1, 2, 2, 2, 0, 1, 2]), non_null_indices: vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9], max_def_level: 2, max_rep_level: 2, @@ -941,8 +1050,8 @@ mod tests { assert_eq!(levels.len(), 1); let expected_levels = ArrayLevels { - def_levels: None, - rep_levels: None, + def_levels: LevelData::Absent, + rep_levels: LevelData::Absent, non_null_indices: (0..10).collect(), max_def_level: 0, max_rep_level: 0, @@ -969,8 +1078,8 @@ mod tests { let logical_nulls = array.logical_nulls(); let expected_levels = ArrayLevels { - def_levels: Some(vec![1, 0, 1, 1, 0]), - rep_levels: None, + def_levels: LevelData::Materialized(vec![1, 0, 1, 1, 0]), + rep_levels: LevelData::Absent, non_null_indices: vec![0, 2, 3], max_def_level: 1, max_rep_level: 0, @@ -1004,8 +1113,8 @@ mod tests { assert_eq!(levels.len(), 1); let expected_levels = ArrayLevels { - def_levels: Some(vec![1; 5]), - rep_levels: Some(vec![0; 5]), + def_levels: LevelData::Materialized(vec![1; 5]), + rep_levels: LevelData::Materialized(vec![0; 5]), non_null_indices: (0..5).collect(), max_def_level: 1, max_rep_level: 1, @@ -1038,8 +1147,8 @@ mod tests { assert_eq!(levels.len(), 1); let expected_levels = ArrayLevels { - def_levels: Some(vec![2, 2, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2]), - rep_levels: Some(vec![0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1]), + def_levels: LevelData::Materialized(vec![2, 2, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2]), + rep_levels: LevelData::Materialized(vec![0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1]), non_null_indices: (0..11).collect(), max_def_level: 2, max_rep_level: 1, @@ -1088,8 +1197,8 @@ mod tests { assert_eq!(levels.len(), 1); let expected_levels = ArrayLevels { - def_levels: Some(vec![0, 2, 0, 3, 3, 3, 3, 3, 3, 3]), - rep_levels: Some(vec![0, 0, 0, 0, 1, 1, 1, 0, 1, 1]), + def_levels: LevelData::Materialized(vec![0, 2, 0, 3, 3, 3, 3, 3, 3, 3]), + rep_levels: LevelData::Materialized(vec![0, 0, 0, 0, 1, 1, 1, 0, 1, 1]), non_null_indices: (4..11).collect(), max_def_level: 3, max_rep_level: 1, @@ -1135,10 +1244,10 @@ mod tests { assert_eq!(levels.len(), 1); let expected_levels = ArrayLevels { - def_levels: Some(vec![ + def_levels: LevelData::Materialized(vec![ 5, 5, 5, 5, 1, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ]), - rep_levels: Some(vec![ + rep_levels: LevelData::Materialized(vec![ 0, 2, 1, 2, 0, 0, 2, 1, 2, 0, 2, 1, 2, 1, 2, 1, 2, 0, 2, 1, 2, 1, 2, ]), non_null_indices: (0..22).collect(), @@ -1177,8 +1286,8 @@ mod tests { assert_eq!(levels.len(), 1); let expected_levels = ArrayLevels { - def_levels: Some(vec![1; 4]), - rep_levels: Some(vec![0; 4]), + def_levels: LevelData::Materialized(vec![1; 4]), + rep_levels: LevelData::Materialized(vec![0; 4]), non_null_indices: (0..4).collect(), max_def_level: 1, max_rep_level: 1, @@ -1210,8 +1319,8 @@ mod tests { assert_eq!(levels.len(), 1); let expected_levels = ArrayLevels { - def_levels: Some(vec![1, 3, 3, 3, 3, 3, 3, 3]), - rep_levels: Some(vec![0, 0, 1, 1, 0, 1, 0, 1]), + def_levels: LevelData::Materialized(vec![1, 3, 3, 3, 3, 3, 3, 3]), + rep_levels: LevelData::Materialized(vec![0, 0, 1, 1, 0, 1, 0, 1]), non_null_indices: (0..7).collect(), max_def_level: 3, max_rep_level: 1, @@ -1259,8 +1368,12 @@ mod tests { assert_eq!(levels.len(), 1); let expected_levels = ArrayLevels { - def_levels: Some(vec![1, 5, 5, 5, 4, 5, 5, 5, 5, 5, 5, 5, 4, 5, 5, 5, 5, 5]), - rep_levels: Some(vec![0, 0, 1, 2, 1, 0, 2, 2, 1, 2, 2, 2, 0, 1, 2, 2, 2, 2]), + def_levels: LevelData::Materialized(vec![ + 1, 5, 5, 5, 4, 5, 5, 5, 5, 5, 5, 5, 4, 5, 5, 5, 5, 5, + ]), + rep_levels: LevelData::Materialized(vec![ + 0, 0, 1, 2, 1, 0, 2, 2, 1, 2, 2, 2, 0, 1, 2, 2, 2, 2, + ]), non_null_indices: (0..15).collect(), max_def_level: 5, max_rep_level: 2, @@ -1300,8 +1413,8 @@ mod tests { let logical_nulls = leaf.logical_nulls(); let expected_levels = ArrayLevels { - def_levels: Some(vec![3, 2, 3, 1, 0, 3]), - rep_levels: None, + def_levels: LevelData::Materialized(vec![3, 2, 3, 1, 0, 3]), + rep_levels: LevelData::Absent, non_null_indices: vec![0, 2, 5], max_def_level: 3, max_rep_level: 0, @@ -1340,8 +1453,8 @@ mod tests { let list_level = &levels[0]; let expected_level = ArrayLevels { - def_levels: Some(vec![0, 3, 3, 3]), - rep_levels: Some(vec![0, 0, 1, 1]), + def_levels: LevelData::Materialized(vec![0, 3, 3, 3]), + rep_levels: LevelData::Materialized(vec![0, 0, 1, 1]), non_null_indices: vec![3, 4, 5], max_def_level: 3, max_rep_level: 1, @@ -1433,8 +1546,8 @@ mod tests { let list_level = &levels[0]; let expected_level = ArrayLevels { - def_levels: None, - rep_levels: None, + def_levels: LevelData::Absent, + rep_levels: LevelData::Absent, non_null_indices: vec![0, 1, 2, 3, 4], max_def_level: 0, max_rep_level: 0, @@ -1448,8 +1561,8 @@ mod tests { let b_logical_nulls = b.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![1, 0, 0, 1, 1]), - rep_levels: None, + def_levels: LevelData::Materialized(vec![1, 0, 0, 1, 1]), + rep_levels: LevelData::Absent, non_null_indices: vec![0, 3, 4], max_def_level: 1, max_rep_level: 0, @@ -1463,8 +1576,8 @@ mod tests { let d_logical_nulls = d.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![1, 1, 1, 2, 1]), - rep_levels: None, + def_levels: LevelData::Materialized(vec![1, 1, 1, 2, 1]), + rep_levels: LevelData::Absent, non_null_indices: vec![3], max_def_level: 2, max_rep_level: 0, @@ -1478,8 +1591,8 @@ mod tests { let f_logical_nulls = f.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![3, 2, 3, 2, 3]), - rep_levels: None, + def_levels: LevelData::Materialized(vec![3, 2, 3, 2, 3]), + rep_levels: LevelData::Absent, non_null_indices: vec![0, 2, 4], max_def_level: 3, max_rep_level: 0, @@ -1586,8 +1699,8 @@ mod tests { let list_level = &levels[0]; let expected_level = ArrayLevels { - def_levels: Some(vec![1; 7]), - rep_levels: Some(vec![0, 1, 0, 1, 0, 1, 1]), + def_levels: LevelData::Materialized(vec![1; 7]), + rep_levels: LevelData::Materialized(vec![0, 1, 0, 1, 0, 1, 1]), non_null_indices: vec![0, 1, 2, 3, 4, 5, 6], max_def_level: 1, max_rep_level: 1, @@ -1601,8 +1714,8 @@ mod tests { let map_values_logical_nulls = map.values().logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![2, 2, 2, 1, 2, 1, 2]), - rep_levels: Some(vec![0, 1, 0, 1, 0, 1, 1]), + def_levels: LevelData::Materialized(vec![2, 2, 2, 1, 2, 1, 2]), + rep_levels: LevelData::Materialized(vec![0, 1, 0, 1, 0, 1, 1]), non_null_indices: vec![0, 1, 2, 4, 6], max_def_level: 2, max_rep_level: 1, @@ -1688,8 +1801,8 @@ mod tests { let logical_nulls = values.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![4, 1, 0, 2, 2, 3, 4]), - rep_levels: Some(vec![0, 0, 0, 0, 1, 0, 0]), + def_levels: LevelData::Materialized(vec![4, 1, 0, 2, 2, 3, 4]), + rep_levels: LevelData::Materialized(vec![0, 0, 0, 0, 1, 0, 0]), non_null_indices: vec![0, 4], max_def_level: 4, max_rep_level: 1, @@ -1730,8 +1843,8 @@ mod tests { let logical_nulls = values.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![4, 4, 3, 2, 0, 4, 4, 0, 1]), - rep_levels: Some(vec![0, 1, 0, 0, 0, 0, 1, 0, 0]), + def_levels: LevelData::Materialized(vec![4, 4, 3, 2, 0, 4, 4, 0, 1]), + rep_levels: LevelData::Materialized(vec![0, 1, 0, 0, 0, 0, 1, 0, 0]), non_null_indices: vec![0, 1, 5, 6], max_def_level: 4, max_rep_level: 1, @@ -1817,8 +1930,8 @@ mod tests { let a1_logical_nulls = a1_values.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![0, 0, 1, 6, 5, 2, 3, 1]), - rep_levels: Some(vec![0, 0, 0, 0, 2, 0, 1, 0]), + def_levels: LevelData::Materialized(vec![0, 0, 1, 6, 5, 2, 3, 1]), + rep_levels: LevelData::Materialized(vec![0, 0, 0, 0, 2, 0, 1, 0]), non_null_indices: vec![1], max_def_level: 6, max_rep_level: 2, @@ -1830,8 +1943,8 @@ mod tests { let a2_logical_nulls = a2_values.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![0, 0, 1, 3, 2, 4, 1]), - rep_levels: Some(vec![0, 0, 0, 0, 0, 1, 0]), + def_levels: LevelData::Materialized(vec![0, 0, 1, 3, 2, 4, 1]), + rep_levels: LevelData::Materialized(vec![0, 0, 0, 0, 0, 1, 0]), non_null_indices: vec![4], max_def_level: 4, max_rep_level: 1, @@ -1870,8 +1983,8 @@ mod tests { let logical_nulls = values.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![0, 0, 3, 3]), - rep_levels: Some(vec![0, 0, 0, 1]), + def_levels: LevelData::Materialized(vec![0, 0, 3, 3]), + rep_levels: LevelData::Materialized(vec![0, 0, 0, 1]), non_null_indices: vec![6, 7], max_def_level: 3, max_rep_level: 1, @@ -2022,8 +2135,8 @@ mod tests { // [[{a: 1}, null], null, [null, null], [{a: null}, {a: 2}]] let values_a_logical_nulls = values_a.logical_nulls(); let expected_a = ArrayLevels { - def_levels: Some(vec![4, 2, 0, 2, 2, 3, 4]), - rep_levels: Some(vec![0, 1, 0, 0, 1, 0, 1]), + def_levels: LevelData::Materialized(vec![4, 2, 0, 2, 2, 3, 4]), + rep_levels: LevelData::Materialized(vec![0, 1, 0, 0, 1, 0, 1]), non_null_indices: vec![0, 7], max_def_level: 4, max_rep_level: 1, @@ -2033,8 +2146,8 @@ mod tests { // [[{b: 2}, null], null, [null, null], [{b: 3}, {b: 4}]] let values_b_logical_nulls = values_b.logical_nulls(); let expected_b = ArrayLevels { - def_levels: Some(vec![3, 2, 0, 2, 2, 3, 3]), - rep_levels: Some(vec![0, 1, 0, 0, 1, 0, 1]), + def_levels: LevelData::Materialized(vec![3, 2, 0, 2, 2, 3, 3]), + rep_levels: LevelData::Materialized(vec![0, 1, 0, 0, 1, 0, 1]), non_null_indices: vec![0, 6, 7], max_def_level: 3, max_rep_level: 1, @@ -2066,8 +2179,8 @@ mod tests { let logical_nulls = values.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![1, 0, 1]), - rep_levels: Some(vec![0, 0, 0]), + def_levels: LevelData::Materialized(vec![1, 0, 1]), + rep_levels: LevelData::Materialized(vec![0, 0, 0]), non_null_indices: vec![], max_def_level: 3, max_rep_level: 1, @@ -2103,8 +2216,8 @@ mod tests { let logical_nulls = values.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![5, 4, 5, 2, 5, 3, 5, 5, 4, 4, 0]), - rep_levels: Some(vec![0, 2, 2, 1, 0, 1, 0, 2, 1, 2, 0]), + def_levels: LevelData::Materialized(vec![5, 4, 5, 2, 5, 3, 5, 5, 4, 4, 0]), + rep_levels: LevelData::Materialized(vec![0, 2, 2, 1, 0, 1, 0, 2, 1, 2, 0]), non_null_indices: vec![0, 2, 3, 4, 5], max_def_level: 5, max_rep_level: 2, @@ -2136,8 +2249,8 @@ mod tests { let logical_nulls = dict.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![0, 0, 1, 1]), - rep_levels: None, + def_levels: LevelData::Materialized(vec![0, 0, 1, 1]), + rep_levels: LevelData::Absent, non_null_indices: vec![2, 3], max_def_level: 1, max_rep_level: 0, @@ -2176,8 +2289,8 @@ mod tests { let array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5, 6])); let logical_nulls = array.logical_nulls(); let levels = ArrayLevels { - def_levels: None, - rep_levels: None, + def_levels: LevelData::Absent, + rep_levels: LevelData::Absent, non_null_indices: vec![0, 1, 2, 3, 4, 5], max_def_level: 0, max_rep_level: 0, @@ -2190,8 +2303,8 @@ mod tests { value_offset: 2, num_values: 3, }); - assert!(sliced.def_levels.is_none()); - assert!(sliced.rep_levels.is_none()); + assert!(matches!(sliced.def_levels, LevelData::Absent)); + assert!(matches!(sliced.rep_levels, LevelData::Absent)); assert_eq!(sliced.non_null_indices, vec![0, 1, 2]); assert_eq!(sliced.array.len(), 3); @@ -2210,8 +2323,8 @@ mod tests { ])); let logical_nulls = array.logical_nulls(); let levels = ArrayLevels { - def_levels: Some(vec![1, 0, 1, 0, 1, 1]), - rep_levels: None, + def_levels: LevelData::Materialized(vec![1, 0, 1, 0, 1, 1]), + rep_levels: LevelData::Absent, non_null_indices: vec![0, 2, 4, 5], max_def_level: 1, max_rep_level: 0, @@ -2224,8 +2337,8 @@ mod tests { value_offset: 1, num_values: 1, }); - assert_eq!(sliced.def_levels, Some(vec![0, 1, 0])); - assert!(sliced.rep_levels.is_none()); + assert_eq!(sliced.def_levels, LevelData::Materialized(vec![0, 1, 0])); + assert!(matches!(sliced.rep_levels, LevelData::Absent)); assert_eq!(sliced.non_null_indices, vec![0]); // [2] shifted by -2 (nni[0]) assert_eq!(sliced.array.len(), 1); } @@ -2264,8 +2377,8 @@ mod tests { ])); let logical_nulls = array.logical_nulls(); let levels = ArrayLevels { - def_levels: Some(vec![3, 0, 3, 2, 0, 3, 3]), - rep_levels: Some(vec![0, 0, 0, 1, 0, 0, 1]), + def_levels: LevelData::Materialized(vec![3, 0, 3, 2, 0, 3, 3]), + rep_levels: LevelData::Materialized(vec![0, 0, 0, 1, 0, 0, 1]), non_null_indices: vec![0, 3, 8, 9], max_def_level: 3, max_rep_level: 1, @@ -2310,8 +2423,8 @@ mod tests { let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None, None, Some(4)])); let logical_nulls = array.logical_nulls(); let levels = ArrayLevels { - def_levels: Some(vec![1, 0, 0, 1]), - rep_levels: None, + def_levels: LevelData::Materialized(vec![1, 0, 0, 1]), + rep_levels: LevelData::Absent, non_null_indices: vec![0, 3], max_def_level: 1, max_rep_level: 0, @@ -2325,7 +2438,7 @@ mod tests { value_offset: 1, num_values: 0, }); - assert_eq!(sliced.def_levels, Some(vec![0, 0])); + assert_eq!(sliced.def_levels, LevelData::Materialized(vec![0, 0])); assert_eq!(sliced.non_null_indices, Vec::::new()); assert_eq!(sliced.array.len(), 0); } diff --git a/parquet/src/arrow/arrow_writer/mod.rs b/parquet/src/arrow/arrow_writer/mod.rs index 641b81257a91..b93e17440120 100644 --- a/parquet/src/arrow/arrow_writer/mod.rs +++ b/parquet/src/arrow/arrow_writer/mod.rs @@ -920,8 +920,11 @@ impl ArrowColumnWriter { chunker: &mut ContentDefinedChunker, ) -> Result<()> { let levels = &col.0; - let chunks = - chunker.get_arrow_chunks(levels.def_levels(), levels.rep_levels(), levels.array())?; + let chunks = chunker.get_arrow_chunks( + levels.def_level_data().as_ref(), + levels.rep_level_data().as_ref(), + levels.array(), + )?; let num_chunks = chunks.len(); for (i, chunk) in chunks.iter().enumerate() { @@ -1384,10 +1387,15 @@ fn write_leaf( } ColumnWriter::BoolColumnWriter(typed) => { let array = column.as_boolean(); - typed.write_batch( - get_bool_array_slice(array, indices).as_slice(), - levels.def_levels(), - levels.rep_levels(), + let values = get_bool_array_slice(array, indices); + typed.write_batch_internal( + values.as_slice(), + None, + levels.def_level_data().as_ref(), + levels.rep_level_data().as_ref(), + None, + None, + None, ) } ColumnWriter::Int64ColumnWriter(typed) => { @@ -1538,7 +1546,15 @@ fn write_leaf( )); } }; - typed.write_batch(bytes.as_slice(), levels.def_levels(), levels.rep_levels()) + typed.write_batch_internal( + bytes.as_slice(), + None, + levels.def_level_data().as_ref(), + levels.rep_level_data().as_ref(), + None, + None, + None, + ) } } } @@ -1551,8 +1567,8 @@ fn write_primitive( writer.write_batch_internal( values, Some(levels.non_null_indices()), - levels.def_levels(), - levels.rep_levels(), + levels.def_level_data().as_ref(), + levels.rep_level_data().as_ref(), None, None, None, diff --git a/parquet/src/column/chunker/cdc.rs b/parquet/src/column/chunker/cdc.rs index cb4328536df3..b40dd74a8d83 100644 --- a/parquet/src/column/chunker/cdc.rs +++ b/parquet/src/column/chunker/cdc.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "arrow")] +use crate::column::writer::LevelDataRef; use crate::errors::{ParquetError, Result}; use crate::file::properties::CdcOptions; use crate::schema::types::ColumnDescriptor; @@ -275,8 +277,8 @@ impl ContentDefinedChunker { /// evaluate if we need to create a new chunk. fn calculate( &mut self, - def_levels: Option<&[i16]>, - rep_levels: Option<&[i16]>, + def_levels: LevelDataRef<'_>, + rep_levels: LevelDataRef<'_>, num_levels: usize, mut roll_value: F, ) -> Vec @@ -322,10 +324,11 @@ impl ContentDefinedChunker { // def_levels: [1, 0, 1, 0, 1] // level: 0 1 2 3 4 // value_offset: 0 1 2 (only increments on def==1) - let def_levels = def_levels.expect("def_levels required when max_def_level > 0"); #[allow(clippy::needless_range_loop)] for offset in 0..num_levels { - let def_level = def_levels[offset]; + let def_level = def_levels + .value_at(offset) + .expect("def_levels required when max_def_level > 0"); self.roll_level(def_level); if def_level == self.max_def_level { // For non-nested data, the leaf array has one slot per @@ -380,13 +383,15 @@ impl ContentDefinedChunker { // Using value_offset=1 would index position 1 (the null slot). // // Using value_offset for roll_value would hash the wrong array slot. - let def_levels = def_levels.expect("def_levels required for nested data"); - let rep_levels = rep_levels.expect("rep_levels required for nested data"); let mut leaf_offset: usize = 0; for offset in 0..num_levels { - let def_level = def_levels[offset]; - let rep_level = rep_levels[offset]; + let def_level = def_levels + .value_at(offset) + .expect("def_levels required for nested data"); + let rep_level = rep_levels + .value_at(offset) + .expect("rep_levels required for nested data"); self.roll_level(def_level); self.roll_level(rep_level); @@ -439,16 +444,20 @@ impl ContentDefinedChunker { #[cfg(feature = "arrow")] pub(crate) fn get_arrow_chunks( &mut self, - def_levels: Option<&[i16]>, - rep_levels: Option<&[i16]>, + def_levels: LevelDataRef<'_>, + rep_levels: LevelDataRef<'_>, array: &dyn arrow_array::Array, ) -> Result> { use arrow_array::cast::AsArray; use arrow_schema::DataType; - let num_levels = match def_levels { - Some(def_levels) => def_levels.len(), - None => array.len(), + // For nested (list) data, null list entries can own non-zero child + // ranges in the leaf array, so `array.len()` may exceed the number of + // levels. Always drive the loop by the level count; fall back to the + // array length only when there are no levels at all. + let num_levels = match (def_levels.len(), rep_levels.len()) { + (0, 0) => array.len(), + (d, r) => d.max(r), }; macro_rules! fixed_width { @@ -566,6 +575,8 @@ impl ContentDefinedChunker { mod tests { use super::*; use crate::basic::Type as PhysicalType; + #[cfg(feature = "arrow")] + use crate::column::writer::LevelDataRef; use crate::schema::types::{ColumnPath, Type}; use std::sync::Arc; @@ -615,9 +626,14 @@ mod tests { // Write a small amount of data — should produce exactly 1 chunk. let num_values = 4; - let chunks = chunker.calculate(None, None, num_values, |c, i| { - c.roll_fixed::<4>(&(i as i32).to_le_bytes()); - }); + let chunks = chunker.calculate( + LevelDataRef::Absent, + LevelDataRef::Absent, + num_values, + |c, i| { + c.roll_fixed::<4>(&(i as i32).to_le_bytes()); + }, + ); assert_eq!(chunks.len(), 1); assert_eq!(chunks[0].level_offset, 0); assert_eq!(chunks[0].value_offset, 0); @@ -636,9 +652,14 @@ mod tests { // Write enough data to exceed max_chunk_size multiple times. // Each i32 = 4 bytes, max_chunk_size=1024, so ~256 values per chunk max. let num_values = 2000; - let chunks = chunker.calculate(None, None, num_values, |c, i| { - c.roll_fixed::<4>(&(i as i32).to_le_bytes()); - }); + let chunks = chunker.calculate( + LevelDataRef::Absent, + LevelDataRef::Absent, + num_values, + |c, i| { + c.roll_fixed::<4>(&(i as i32).to_le_bytes()); + }, + ); // Should have multiple chunks assert!(chunks.len() > 1); @@ -668,10 +689,10 @@ mod tests { }; let mut chunker1 = ContentDefinedChunker::new(&make_desc(0, 0), &options).unwrap(); - let chunks1 = chunker1.calculate(None, None, 200, roll); + let chunks1 = chunker1.calculate(LevelDataRef::Absent, LevelDataRef::Absent, 200, roll); let mut chunker2 = ContentDefinedChunker::new(&make_desc(0, 0), &options).unwrap(); - let chunks2 = chunker2.calculate(None, None, 200, roll); + let chunks2 = chunker2.calculate(LevelDataRef::Absent, LevelDataRef::Absent, 200, roll); assert_eq!(chunks1.len(), chunks2.len()); for (a, b) in chunks1.iter().zip(chunks2.iter()) { @@ -699,9 +720,14 @@ mod tests { .collect(); let expected_non_null: usize = def_levels.iter().filter(|&&d| d == 1).count(); - let chunks = chunker.calculate(Some(&def_levels), None, num_levels, |c, i| { - c.roll_fixed::<4>(&(i as i32).to_le_bytes()); - }); + let chunks = chunker.calculate( + LevelDataRef::Materialized(&def_levels), + LevelDataRef::Absent, + num_levels, + |c, i| { + c.roll_fixed::<4>(&(i as i32).to_le_bytes()); + }, + ); assert!(!chunks.is_empty()); let total_levels: usize = chunks.iter().map(|c| c.num_levels).sum(); @@ -729,6 +755,7 @@ mod arrow_tests { use crate::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use crate::arrow::arrow_writer::ArrowWriter; + use crate::column::writer::LevelDataRef; use crate::file::properties::{CdcOptions, WriterProperties}; use crate::file::reader::{FileReader, SerializedFileReader}; @@ -1109,6 +1136,27 @@ mod arrow_tests { buf } + #[test] + fn cdc_all_null_arrow_column_writes_data_pages() { + let array = Arc::new(Int32Array::from(vec![None::; 4096])) as ArrayRef; + let schema = Arc::new(Schema::new(vec![Field::new("f0", DataType::Int32, true)])); + let batch = RecordBatch::try_new(schema, vec![array.clone()]).unwrap(); + + let data = write_with_cdc_options(&[&batch], 64, 256, Some(4096), false); + let info = get_column_info(&data, 0); + + assert_eq!(info.len(), 1); + assert!( + !info[0].page_lengths.is_empty(), + "all-null CDC write should still emit data pages" + ); + assert_eq!( + info[0].page_lengths.iter().sum::(), + array.len() as i64, + "all-null CDC pages should account for every input row" + ); + } + fn read_batches(data: &[u8]) -> Vec { let reader = ParquetRecordBatchReaderBuilder::try_new(bytes::Bytes::from(data.to_vec())) .unwrap() @@ -2190,11 +2238,15 @@ mod arrow_tests { let array: Int32Array = (0..n).map(|i| test_hash(0, i as u64) as i32).collect(); let mut chunker = super::ContentDefinedChunker::new(&desc, &options).unwrap(); - let chunks = chunker.get_arrow_chunks(None, None, &array).unwrap(); + let chunks = chunker + .get_arrow_chunks(LevelDataRef::Absent, LevelDataRef::Absent, &array) + .unwrap(); let sliced = array.slice(offset, n - offset); let mut chunker2 = super::ContentDefinedChunker::new(&desc, &options).unwrap(); - let chunks2 = chunker2.get_arrow_chunks(None, None, &sliced).unwrap(); + let chunks2 = chunker2 + .get_arrow_chunks(LevelDataRef::Absent, LevelDataRef::Absent, &sliced) + .unwrap(); let values: Vec = chunks.iter().map(|c| c.num_values).collect(); let values2: Vec = chunks2.iter().map(|c| c.num_values).collect(); diff --git a/parquet/src/column/writer/mod.rs b/parquet/src/column/writer/mod.rs index f755beed55c4..5d14ac6856f9 100644 --- a/parquet/src/column/writer/mod.rs +++ b/parquet/src/column/writer/mod.rs @@ -314,6 +314,67 @@ impl ColumnMetrics { } } +/// Borrowed view of level data, analogous to `&str` for `LevelData`'s `String`. +/// +/// `LevelDataRef` can be constructed from `LevelData` and directly from an existing +/// `&[i16]` without allocating. +/// +/// The variants are different physical representations of the same logical +/// sequence of levels. +#[derive(Debug, Clone, Copy)] +pub(crate) enum LevelDataRef<'a> { + Absent, + Materialized(&'a [i16]), + Uniform { value: i16, count: usize }, +} + +impl<'a> From<&'a [i16]> for LevelDataRef<'a> { + fn from(levels: &'a [i16]) -> Self { + Self::Materialized(levels) + } +} + +impl<'a> From> for LevelDataRef<'a> { + fn from(levels: Option<&'a [i16]>) -> Self { + levels.map_or(Self::Absent, Self::from) + } +} + +impl<'a> LevelDataRef<'a> { + pub(crate) fn len(self) -> usize { + match self { + Self::Absent => 0, + Self::Materialized(values) => values.len(), + Self::Uniform { count, .. } => count, + } + } + + pub(crate) fn first(self) -> Option { + match self { + Self::Absent => None, + Self::Materialized(values) => values.first().copied(), + Self::Uniform { value, count } => (count > 0).then_some(value), + } + } + + #[cfg(feature = "arrow")] + pub(crate) fn value_at(self, idx: usize) -> Option { + match self { + Self::Absent => None, + Self::Materialized(values) => values.get(idx).copied(), + Self::Uniform { value, count } => (idx < count).then_some(value), + } + } + + pub(crate) fn slice(self, offset: usize, len: usize) -> Self { + match self { + Self::Absent => Self::Absent, + Self::Materialized(values) => Self::Materialized(&values[offset..offset + len]), + Self::Uniform { value, .. } => Self::Uniform { value, count: len }, + } + } +} + /// Typed column writer for a primitive column. pub type ColumnWriterImpl<'a, T> = GenericColumnWriter<'a, ColumnValueEncoderImpl>; @@ -424,21 +485,19 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { &mut self, values: &E::Values, value_indices: Option<&[usize]>, - def_levels: Option<&[i16]>, - rep_levels: Option<&[i16]>, + def_levels: LevelDataRef<'_>, + rep_levels: LevelDataRef<'_>, min: Option<&E::T>, max: Option<&E::T>, distinct_count: Option, ) -> Result { // Check if number of definition levels is the same as number of repetition levels. - if let (Some(def), Some(rep)) = (def_levels, rep_levels) { - if def.len() != rep.len() { - return Err(general_err!( - "Inconsistent length of definition and repetition levels: {} != {}", - def.len(), - rep.len() - )); - } + if def_levels.len() != 0 && rep_levels.len() != 0 && def_levels.len() != rep_levels.len() { + return Err(general_err!( + "Inconsistent length of definition and repetition levels: {} != {}", + def_levels.len(), + rep_levels.len() + )); } // We check for DataPage limits only after we have inserted the values. If a user @@ -450,10 +509,11 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { // TODO: find out why we don't account for size of levels when we estimate page // size. - - let num_levels = match def_levels { - Some(def_levels) => def_levels.len(), - None => values.len(), + let num_levels = def_levels.len().max(rep_levels.len()); + let num_levels = if num_levels > 0 { + num_levels + } else { + value_indices.map_or(values.len(), |i| i.len()) }; if let Some(min) = min { @@ -472,13 +532,24 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { let mut values_offset = 0; let mut levels_offset = 0; - let base_batch_size = self.props.write_batch_size(); + let both_levels_compact = !matches!(def_levels, LevelDataRef::Materialized(_)) + && !matches!(rep_levels, LevelDataRef::Materialized(_)); + let has_levels = !matches!(def_levels, LevelDataRef::Absent) + || !matches!(rep_levels, LevelDataRef::Absent); + // When both level vectors are compact (Uniform or Absent), there is no + // materialized slice to split and the per-mini-batch work is O(1), so we + // can safely use a much larger batch size. + let base_batch_size = if both_levels_compact && has_levels { + self.props.data_page_row_count_limit() + } else { + self.props.write_batch_size() + }; while levels_offset < num_levels { let mut end_offset = num_levels.min(levels_offset + base_batch_size); // Split at record boundary - if let Some(r) = rep_levels { - while end_offset < r.len() && r[end_offset] != 0 { + if let LevelDataRef::Materialized(levels) = rep_levels { + while end_offset < levels.len() && levels[end_offset] != 0 { end_offset += 1; } } @@ -488,8 +559,8 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { values_offset, value_indices, end_offset - levels_offset, - def_levels.map(|lv| &lv[levels_offset..end_offset]), - rep_levels.map(|lv| &lv[levels_offset..end_offset]), + def_levels.slice(levels_offset, end_offset - levels_offset), + rep_levels.slice(levels_offset, end_offset - levels_offset), )?; levels_offset = end_offset; } @@ -516,7 +587,15 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { def_levels: Option<&[i16]>, rep_levels: Option<&[i16]>, ) -> Result { - self.write_batch_internal(values, None, def_levels, rep_levels, None, None, None) + self.write_batch_internal( + values, + None, + LevelDataRef::from(def_levels), + LevelDataRef::from(rep_levels), + None, + None, + None, + ) } /// Writer may optionally provide pre-calculated statistics for use when computing @@ -538,8 +617,8 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { self.write_batch_internal( values, None, - def_levels, - rep_levels, + LevelDataRef::from(def_levels), + LevelDataRef::from(rep_levels), min, max, distinct_count, @@ -650,32 +729,53 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { values_offset: usize, value_indices: Option<&[usize]>, num_levels: usize, - def_levels: Option<&[i16]>, - rep_levels: Option<&[i16]>, + def_levels: LevelDataRef<'_>, + rep_levels: LevelDataRef<'_>, ) -> Result { // Process definition levels and determine how many values to write. let values_to_write = if self.descr.max_def_level() > 0 { - let levels = def_levels.ok_or_else(|| { - general_err!( - "Definition levels are required, because max definition level = {}", - self.descr.max_def_level() - ) - })?; - - let mut values_to_write = 0usize; let max_def = self.descr.max_def_level(); - let encoder = &mut self.def_levels_encoder; - match self.page_metrics.definition_level_histogram.as_mut() { - Some(histogram) => encoder.put_with_observer(levels, |level, count| { - values_to_write += count * (level == max_def) as usize; - histogram.increment_by(level, count as i64); - }), - None => encoder.put_with_observer(levels, |level, count| { - values_to_write += count * (level == max_def) as usize; - }), - }; - self.page_metrics.num_page_nulls += (levels.len() - values_to_write) as u64; - values_to_write + match def_levels { + LevelDataRef::Absent => { + return Err(general_err!( + "Definition levels are required, because max definition level = {}", + self.descr.max_def_level() + )); + } + LevelDataRef::Materialized(levels) => { + // General path for caller-provided or already-materialized + // level buffers. + let mut values_to_write = 0usize; + let encoder = &mut self.def_levels_encoder; + match self.page_metrics.definition_level_histogram.as_mut() { + Some(histogram) => encoder.put_with_observer(levels, |level, count| { + values_to_write += count * (level == max_def) as usize; + histogram.increment_by(level, count as i64); + }), + None => encoder.put_with_observer(levels, |level, count| { + values_to_write += count * (level == max_def) as usize; + }), + }; + self.page_metrics.num_page_nulls += (levels.len() - values_to_write) as u64; + values_to_write + } + LevelDataRef::Uniform { value, count } => { + // Fast path for all-null, all-valid, or otherwise uniform + // definition levels without materializing a level buffer. + let encoder = &mut self.def_levels_encoder; + match self.page_metrics.definition_level_histogram.as_mut() { + Some(histogram) => { + encoder.put_n_with_observer(value, count, |level, run_len| { + histogram.increment_by(level, run_len as i64); + }) + } + None => encoder.put_n_with_observer(value, count, |_, _| {}), + }; + let values_to_write = count * (value == max_def) as usize; + self.page_metrics.num_page_nulls += (count - values_to_write) as u64; + values_to_write + } + } } else { num_levels }; @@ -683,31 +783,50 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { // Process repetition levels and determine how many rows we are about to process. if self.descr.max_rep_level() > 0 { // A row could contain more than one value. - let levels = rep_levels.ok_or_else(|| { + let first_level = rep_levels.first().ok_or_else(|| { general_err!( "Repetition levels are required, because max repetition level = {}", self.descr.max_rep_level() ) })?; - if !levels.is_empty() && levels[0] != 0 { + if first_level != 0 { return Err(general_err!( "Write must start at a record boundary, got non-zero repetition level of {}", - levels[0] + first_level )); } let mut new_rows = 0u32; - let encoder = &mut self.rep_levels_encoder; - match self.page_metrics.repetition_level_histogram.as_mut() { - Some(histogram) => encoder.put_with_observer(levels, |level, count| { - new_rows += (count as u32) * (level == 0) as u32; - histogram.increment_by(level, count as i64); - }), - None => encoder.put_with_observer(levels, |level, count| { - new_rows += (count as u32) * (level == 0) as u32; - }), - }; + match rep_levels { + LevelDataRef::Absent => unreachable!(), + LevelDataRef::Materialized(levels) => { + let encoder = &mut self.rep_levels_encoder; + match self.page_metrics.repetition_level_histogram.as_mut() { + Some(histogram) => encoder.put_with_observer(levels, |level, count| { + new_rows += (count as u32) * (level == 0) as u32; + histogram.increment_by(level, count as i64); + }), + None => encoder.put_with_observer(levels, |level, count| { + new_rows += (count as u32) * (level == 0) as u32; + }), + }; + } + LevelDataRef::Uniform { value, count } => { + let encoder = &mut self.rep_levels_encoder; + match self.page_metrics.repetition_level_histogram.as_mut() { + Some(histogram) => { + encoder.put_n_with_observer(value, count, |level, run_len| { + new_rows += (run_len as u32) * (level == 0) as u32; + histogram.increment_by(level, run_len as i64); + }) + } + None => encoder.put_n_with_observer(value, count, |level, run_len| { + new_rows += (run_len as u32) * (level == 0) as u32; + }), + }; + } + } self.page_metrics.num_buffered_rows += new_rows; } else { // Each value is exactly one row. @@ -4451,4 +4570,221 @@ mod tests { result.metadata.compressed_size() ); } + + struct ColumnRoundTripUniform<'a, T: DataType> { + props: WriterProperties, + values: &'a [T::T], + def_levels: LevelDataRef<'a>, + rep_levels: LevelDataRef<'a>, + max_def_level: i16, + max_rep_level: i16, + expected_values: &'a [T::T], + expected_def_levels: Option<&'a [i16]>, + expected_rep_levels: Option<&'a [i16]>, + } + + impl<'a, T: DataType> ColumnRoundTripUniform<'a, T> + where + T::T: PartialEq + std::fmt::Debug, + { + fn new() -> Self { + Self { + props: Default::default(), + values: &[], + def_levels: LevelDataRef::Absent, + rep_levels: LevelDataRef::Absent, + max_def_level: 0, + max_rep_level: 0, + expected_values: &[], + expected_def_levels: None, + expected_rep_levels: None, + } + } + + fn with_props(mut self, props: WriterProperties) -> Self { + self.props = props; + self + } + + fn with_values(mut self, values: &'a [T::T]) -> Self { + self.values = values; + self + } + + fn with_def_levels(mut self, def_levels: LevelDataRef<'a>) -> Self { + self.def_levels = def_levels; + self + } + + fn with_rep_levels(mut self, rep_levels: LevelDataRef<'a>) -> Self { + self.rep_levels = rep_levels; + self + } + + fn with_max_def_level(mut self, max_def_level: i16) -> Self { + self.max_def_level = max_def_level; + self + } + + fn with_max_rep_level(mut self, max_rep_level: i16) -> Self { + self.max_rep_level = max_rep_level; + self + } + + fn with_expected_values(mut self, expected_values: &'a [T::T]) -> Self { + self.expected_values = expected_values; + self + } + + fn with_expected_def_levels(mut self, expected_def_levels: &'a [i16]) -> Self { + self.expected_def_levels = Some(expected_def_levels); + self + } + + fn with_expected_rep_levels(mut self, expected_rep_levels: &'a [i16]) -> Self { + self.expected_rep_levels = Some(expected_rep_levels); + self + } + + /// Write-then-read roundtrip using `write_batch_internal` with the given + /// [`LevelDataRef`] variants, and assert the read-back matches `expected_*`. + fn run(self) { + let mut file = tempfile::tempfile().unwrap(); + let mut write = TrackedWrite::new(&mut file); + let page_writer = Box::new(SerializedPageWriter::new(&mut write)); + let mut writer = get_test_column_writer::( + page_writer, + self.max_def_level, + self.max_rep_level, + Arc::new(self.props), + ); + + writer + .write_batch_internal( + self.values, + None, + self.def_levels, + self.rep_levels, + None, + None, + None, + ) + .unwrap(); + let result = writer.close().unwrap(); + drop(write); + + let props = ReaderProperties::builder() + .set_backward_compatible_lz4(false) + .build(); + let page_reader = Box::new( + SerializedPageReader::new_with_properties( + Arc::new(file), + &result.metadata, + result.rows_written as usize, + None, + Arc::new(props), + ) + .unwrap(), + ); + let mut reader = + get_test_column_reader::(page_reader, self.max_def_level, self.max_rep_level); + + let batch_size = self + .expected_def_levels + .map_or(self.expected_values.len(), |l| l.len()); + let mut actual_values = Vec::with_capacity(batch_size); + let mut actual_def = self + .expected_def_levels + .map(|_| Vec::with_capacity(batch_size)); + let mut actual_rep = self + .expected_rep_levels + .map(|_| Vec::with_capacity(batch_size)); + + let (_, values_read, levels_read) = reader + .read_records( + batch_size, + actual_def.as_mut(), + actual_rep.as_mut(), + &mut actual_values, + ) + .unwrap(); + + assert_eq!(&actual_values[..values_read], self.expected_values); + if let Some(ref v) = actual_def { + assert_eq!(&v[..levels_read], self.expected_def_levels.unwrap()); + } + if let Some(ref v) = actual_rep { + assert_eq!(&v[..levels_read], self.expected_rep_levels.unwrap()); + } + } + } + + #[test] + fn test_uniform_def_levels_all_null() { + // All-null column: def_level=0 (null) for every slot, no values written. + let max_def_level = 1; + let count = 100; + let expected_def_levels = vec![0i16; count]; + ColumnRoundTripUniform::::new() + .with_def_levels(LevelDataRef::Uniform { value: 0, count }) + .with_max_def_level(max_def_level) + .with_expected_def_levels(&expected_def_levels) + .run(); + } + + #[test] + fn test_uniform_def_levels_all_valid() { + // All-valid column: def_level=max for every slot, all values written. + let max_def_level = 1; + let values: Vec = (0..50).collect(); + let expected_def_levels = vec![max_def_level; values.len()]; + ColumnRoundTripUniform::::new() + .with_values(&values) + .with_def_levels(LevelDataRef::Uniform { + value: max_def_level, + count: values.len(), + }) + .with_max_def_level(max_def_level) + .with_expected_values(&values) + .with_expected_def_levels(&expected_def_levels) + .run(); + } + + #[test] + fn test_uniform_def_and_rep_levels() { + // Simulates a list column where every row is null: + // def=0, rep=0 for each row (one row = one entry with no child values). + let max_def_level = 2; + let max_rep_level = 1; + let count = 200; + let expected_def_levels = vec![0i16; count]; + let expected_rep_levels = vec![0i16; count]; + ColumnRoundTripUniform::::new() + .with_def_levels(LevelDataRef::Uniform { value: 0, count }) + .with_rep_levels(LevelDataRef::Uniform { value: 0, count }) + .with_max_def_level(max_def_level) + .with_max_rep_level(max_rep_level) + .with_expected_def_levels(&expected_def_levels) + .with_expected_rep_levels(&expected_rep_levels) + .run(); + } + + #[test] + fn test_uniform_levels_v1_and_v2() { + // Verify uniform levels work identically for both Parquet writer versions. + for version in [WriterVersion::PARQUET_1_0, WriterVersion::PARQUET_2_0] { + let props = WriterProperties::builder() + .set_writer_version(version) + .build(); + let max_def = 1; + let count = 100; + let expected_def_levels = vec![0i16; count]; + ColumnRoundTripUniform::::new() + .with_props(props) + .with_def_levels(LevelDataRef::Uniform { value: 0, count }) + .with_max_def_level(max_def) + .with_expected_def_levels(&expected_def_levels) + .run(); + } + } } diff --git a/parquet/src/encodings/levels.rs b/parquet/src/encodings/levels.rs index 6b6de8b0e1c6..841afd8b3186 100644 --- a/parquet/src/encodings/levels.rs +++ b/parquet/src/encodings/levels.rs @@ -92,6 +92,37 @@ impl LevelEncoder { } } + /// Encode `count` repetitions of a single level value, calling + /// `observer(value, count)` exactly once. + /// + /// This can be used to encode uniform runs without allocating a level + /// buffer. Like [`Self::put_with_observer`], but specialized for a single + /// repeated level value. + /// + /// This is O(1) amortized for RLE-based encoders (after a small warmup). + #[inline] + pub fn put_n_with_observer(&mut self, value: i16, count: usize, mut observer: F) + where + F: FnMut(i16, usize), + { + let encoder = match *self { + LevelEncoder::Rle(ref mut encoder) | LevelEncoder::RleV2(ref mut encoder) => encoder, + }; + + // Feed values individually until the encoder enters RLE accumulation + // mode for this value, or until we've encoded everything. + let mut remaining = count; + while remaining > 0 && !encoder.is_accumulating_rle(value as u64) { + encoder.put(value as u64); + remaining -= 1; + } + // If we're now in accumulation mode, bulk-extend the rest. + if remaining > 0 { + encoder.extend_run(remaining); + } + observer(value, count); + } + /// Finalizes level encoder, flush all intermediate buffers and return resulting /// encoded buffer. Returned buffer is already truncated to encoded bytes only. #[inline] @@ -139,3 +170,116 @@ impl LevelEncoder { result } } + +#[cfg(test)] +mod tests { + use super::*; + + /// Encode `count` repetitions of `value` using `put_with_observer` and + /// return the raw encoded bytes. + fn reference_encode(max_level: i16, values: &[i16]) -> Vec { + let mut enc = LevelEncoder::v2_streaming(max_level); + enc.put_with_observer(values, |_, _| {}); + enc.consume() + } + + #[test] + fn test_put_n_with_observer_large_run() { + // Large count exercises the bulk extend_run path (past the 8-value warmup). + let max_level = 3; + let count = 10_000; + let mut enc = LevelEncoder::v2_streaming(max_level); + enc.put_n_with_observer(2, count, |_, _| {}); + assert_eq!(enc.consume(), reference_encode(max_level, &vec![2; count])); + } + + #[test] + fn test_put_n_with_observer_small_count() { + // Count smaller than the RLE warmup threshold — only the per-element loop runs. + let max_level = 3; + let mut enc = LevelEncoder::v2_streaming(max_level); + enc.put_n_with_observer(1, 5, |_, _| {}); + assert_eq!(enc.consume(), reference_encode(max_level, &[1; 5])); + } + + #[test] + fn test_put_n_with_observer_exact_threshold() { + // Exactly 8 values: the warmup loop completes and extend_run gets 0. + let max_level = 3; + let mut enc = LevelEncoder::v2_streaming(max_level); + enc.put_n_with_observer(3, 8, |_, _| {}); + assert_eq!(enc.consume(), reference_encode(max_level, &[3; 8])); + } + + #[test] + fn test_put_n_with_observer_single_value() { + let max_level = 1; + let mut enc = LevelEncoder::v2_streaming(max_level); + enc.put_n_with_observer(1, 1, |_, _| {}); + assert_eq!(enc.consume(), reference_encode(max_level, &[1])); + } + + #[test] + fn test_put_n_with_observer_zero_count() { + let max_level = 3; + let mut enc = LevelEncoder::v2_streaming(max_level); + enc.put_n_with_observer(2, 0, |_, _| {}); + assert_eq!(enc.consume(), reference_encode(max_level, &[])); + } + + #[test] + fn test_put_n_with_observer_calls_observer_exactly_once() { + let mut enc = LevelEncoder::v2_streaming(3); + let mut calls: Vec<(i16, usize)> = Vec::new(); + enc.put_n_with_observer(2, 500, |val, cnt| calls.push((val, cnt))); + assert_eq!(calls, vec![(2, 500)]); + } + + #[test] + fn test_put_n_with_observer_zero_count_calls_observer() { + let mut enc = LevelEncoder::v2_streaming(3); + let mut calls: Vec<(i16, usize)> = Vec::new(); + enc.put_n_with_observer(1, 0, |val, cnt| calls.push((val, cnt))); + assert_eq!(calls, vec![(1, 0)]); + } + + #[test] + fn test_put_n_with_observer_followed_by_different_value() { + // Two consecutive put_n calls with different values — verifies that + // the encoder correctly transitions between runs. + let max_level = 3; + let mut enc = LevelEncoder::v2_streaming(max_level); + let mut calls: Vec<(i16, usize)> = Vec::new(); + enc.put_n_with_observer(1, 100, |v, c| calls.push((v, c))); + enc.put_n_with_observer(3, 200, |v, c| calls.push((v, c))); + assert_eq!(calls, vec![(1, 100), (3, 200)]); + + let reference = reference_encode(max_level, &[&[1i16; 100][..], &[3i16; 200]].concat()); + assert_eq!(enc.consume(), reference); + } + + #[test] + fn test_put_n_with_observer_interleaved_with_put_with_observer() { + // Mix put_n_with_observer and put_with_observer to verify they compose. + let max_level = 3; + let mut enc = LevelEncoder::v2_streaming(max_level); + enc.put_n_with_observer(2, 50, |_, _| {}); + enc.put_with_observer(&[0, 0, 1, 1, 3], |_, _| {}); + enc.put_n_with_observer(2, 50, |_, _| {}); + + let input = [&[2i16; 50][..], &[0, 0, 1, 1, 3], &[2i16; 50]].concat(); + assert_eq!(enc.consume(), reference_encode(max_level, &input)); + } + + #[test] + fn test_put_n_with_observer_v1_roundtrip() { + // Also verify V1 (Rle variant with length header) works correctly. + let max_level = 3; + let mut enc = LevelEncoder::v1_streaming(max_level); + enc.put_n_with_observer(2, 1000, |_, _| {}); + + let mut ref_enc = LevelEncoder::v1_streaming(max_level); + ref_enc.put_with_observer(&[2; 1000], |_, _| {}); + assert_eq!(enc.consume(), ref_enc.consume()); + } +} From c025c48f284f1c1829cd1c469ac0232d0ec9c79f Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Thu, 7 May 2026 10:17:59 -0700 Subject: [PATCH 21/33] [Parquet]: GH-563: Make `path_in_schema` optional (#9678) # Which issue does this PR close? none # Rationale for this change This is a proof of concept implementation for https://github.com/apache/parquet-format/issues/563 # What changes are included in this PR? Since version 57.0.0, this crate has been tolerant of a missing `path_in_schema`. This PR adds options to cease writing the field as well. The option defaults to continuing to write the field. See related discussion on parquet mailing list: https://lists.apache.org/thread/czm2bk45wwtkhhpqxqvmx9dk5wkwk1kt # Are these changes tested? Yes # Are there any user-facing changes? No, this only adds an optional behavior change that defaults to no change # Related PRs - https://github.com/apache/parquet-format/issues/563 - https://github.com/apache/parquet-format/pull/564 - https://github.com/apache/parquet-java/pull/3470 --- parquet/benches/metadata.rs | 20 ++++++++-- parquet/src/arrow/arrow_writer/mod.rs | 38 ++++++++++++++++++ parquet/src/bin/parquet-rewrite.rs | 7 ++++ parquet/src/file/metadata/thrift/mod.rs | 17 ++++++-- parquet/src/file/metadata/writer.rs | 21 +++++++++- parquet/src/file/properties.rs | 52 +++++++++++++++++++++++++ parquet/src/file/writer.rs | 2 + parquet/src/parquet_thrift.rs | 19 ++++++++- 8 files changed, 166 insertions(+), 10 deletions(-) diff --git a/parquet/benches/metadata.rs b/parquet/benches/metadata.rs index c9a6cf3b762c..750e33a2d364 100644 --- a/parquet/benches/metadata.rs +++ b/parquet/benches/metadata.rs @@ -41,7 +41,7 @@ use parquet::file::serialized_reader::ReadOptionsBuilder; const NUM_COLUMNS: usize = 10_000; const NUM_ROW_GROUPS: usize = 10; -fn encoded_meta(is_nullable: bool, has_lists: bool) -> Vec { +fn encoded_meta(is_nullable: bool, has_lists: bool, write_path_in_schema: bool) -> Vec { let mut rng = seedable_rng(); let mut column_desc_ptrs: Vec = Vec::with_capacity(NUM_COLUMNS); @@ -143,7 +143,11 @@ fn encoded_meta(is_nullable: bool, has_lists: bool) -> Vec { let mut buffer = Vec::with_capacity(1024); { let buf = TrackedWrite::new(&mut buffer); - let writer = ParquetMetaDataWriter::new_with_tracked(buf, &metadata); + let mut writer = ParquetMetaDataWriter::new_with_tracked(buf, &metadata); + // use defaults unless `write_path_in_schema` is false + if !write_path_in_schema { + writer = writer.with_write_path_in_schema(write_path_in_schema); + } writer.finish().unwrap(); } @@ -233,7 +237,7 @@ fn criterion_benchmark(c: &mut Criterion) { }) }); - let buf: Bytes = black_box(encoded_meta(false, false)).into(); + let buf: Bytes = black_box(encoded_meta(false, false, true)).into(); let options = ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false); c.bench_function("decode parquet metadata (wide)", |b| { b.iter(|| { @@ -275,7 +279,15 @@ fn criterion_benchmark(c: &mut Criterion) { }) }); - let buf: Bytes = black_box(encoded_meta(true, true)).into(); + let buf: Bytes = black_box(encoded_meta(false, false, false)).into(); + let options = ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false); + c.bench_function("decode parquet metadata no path_in_schema (wide)", |b| { + b.iter(|| { + ParquetMetaDataReader::decode_metadata_with_options(&buf, Some(&options)).unwrap(); + }) + }); + + let buf: Bytes = black_box(encoded_meta(true, true, true)).into(); c.bench_function("decode parquet metadata w/ size stats (wide)", |b| { b.iter(|| { ParquetMetaDataReader::decode_metadata(&buf).unwrap(); diff --git a/parquet/src/arrow/arrow_writer/mod.rs b/parquet/src/arrow/arrow_writer/mod.rs index b93e17440120..d1f1a3cffe14 100644 --- a/parquet/src/arrow/arrow_writer/mod.rs +++ b/parquet/src/arrow/arrow_writer/mod.rs @@ -4596,6 +4596,44 @@ mod tests { } } + #[test] + fn test_arrow_writer_skip_path_in_schema() { + let batch_schema = Schema::new(vec![Field::new("int32", DataType::Int32, false)]); + let file_schema = Arc::new(batch_schema.clone()); + + let batch = RecordBatch::try_new( + Arc::new(batch_schema), + vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4])) as _], + ) + .unwrap(); + + // default options should still write path_in_schema + let skip_options = ArrowWriterOptions::new(); + + let mut buf = Vec::with_capacity(1024); + let mut writer = + ArrowWriter::try_new_with_options(&mut buf, file_schema.clone(), skip_options).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + // override to not write path_in_schema + let skip_options = ArrowWriterOptions::new().with_properties( + WriterProperties::builder() + .set_write_path_in_schema(false) + .build(), + ); + + let mut buf2 = Vec::with_capacity(1024); + let mut writer = + ArrowWriter::try_new_with_options(&mut buf2, file_schema.clone(), skip_options) + .unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + // buf2 should be a bit smaller due to lack of path_in_schema + assert!(buf.len() > buf2.len()); + } + #[test] fn mismatched_schemas() { let batch_schema = Schema::new(vec![Field::new("count", DataType::Int32, false)]); diff --git a/parquet/src/bin/parquet-rewrite.rs b/parquet/src/bin/parquet-rewrite.rs index 5bdd432be530..7637f26d77ed 100644 --- a/parquet/src/bin/parquet-rewrite.rs +++ b/parquet/src/bin/parquet-rewrite.rs @@ -279,6 +279,10 @@ struct Args { #[clap(long)] write_page_header_statistics: Option, + /// Write path_in_schema to the column metadata. + #[clap(long)] + write_path_in_schema: Option, + /// Sets whether bloom filter is enabled for all columns. #[clap(long)] bloom_filter_enabled: Option, @@ -406,6 +410,9 @@ fn main() { if let Some(value) = args.coerce_types { writer_properties_builder = writer_properties_builder.set_coerce_types(value); } + if let Some(value) = args.write_path_in_schema { + writer_properties_builder = writer_properties_builder.set_write_path_in_schema(value); + } if let Some(value) = args.write_batch_size { writer_properties_builder = writer_properties_builder.set_write_batch_size(value); } diff --git a/parquet/src/file/metadata/thrift/mod.rs b/parquet/src/file/metadata/thrift/mod.rs index ad8f6312c215..8c1147ff2c88 100644 --- a/parquet/src/file/metadata/thrift/mod.rs +++ b/parquet/src/file/metadata/thrift/mod.rs @@ -1333,10 +1333,15 @@ pub(super) fn serialize_column_meta_data( .encodings() .collect::>() .write_thrift_field(w, 2, 1)?; - let path = column_chunk.column_descr.path().parts(); - let path: Vec<&str> = path.iter().map(|v| v.as_str()).collect(); - path.write_thrift_field(w, 3, 2)?; - column_chunk.compression.write_thrift_field(w, 4, 3)?; + if w.write_path_in_schema() { + let path = column_chunk.column_descr.path().parts(); + let path: Vec<&str> = path.iter().map(|v| v.as_str()).collect(); + path.write_thrift_field(w, 3, 2)?; + column_chunk.compression.write_thrift_field(w, 4, 3)?; + } else { + column_chunk.compression.write_thrift_field(w, 4, 2)?; + } + column_chunk.num_values.write_thrift_field(w, 5, 4)?; column_chunk .total_uncompressed_size @@ -1406,6 +1411,8 @@ pub(super) fn serialize_column_meta_data( pub(super) struct FileMeta<'a> { pub(super) file_metadata: &'a crate::file::metadata::FileMetaData, pub(super) row_groups: &'a Vec, + // If true, then write the `path_in_schema` field in the ColumnMetaData struct. + pub(super) write_path_in_schema: bool, } // struct FileMetaData { @@ -1425,6 +1432,8 @@ impl<'a> WriteThrift for FileMeta<'a> { // needed for last_field_id w/o encryption #[allow(unused_assignments)] fn write_thrift(&self, writer: &mut ThriftCompactOutputProtocol) -> Result<()> { + writer.set_write_path_in_schema(self.write_path_in_schema); + self.file_metadata .version .write_thrift_field(writer, 1, 0)?; diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index 275b4ff28e56..cd5d617f9372 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -62,6 +62,7 @@ pub(crate) struct ThriftMetadataWriter<'a, W: Write> { created_by: Option, object_writer: MetadataObjectWriter, writer_version: i32, + write_path_in_schema: bool, } impl<'a, W: Write> ThriftMetadataWriter<'a, W> { @@ -259,6 +260,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { let file_meta = FileMeta { file_metadata: &file_metadata, row_groups: &row_groups, + write_path_in_schema: self.write_path_in_schema, }; // Write file metadata @@ -293,6 +295,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { row_groups: Vec, created_by: Option, writer_version: i32, + write_path_in_schema: bool, ) -> Self { Self { buf, @@ -304,6 +307,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { created_by, object_writer: Default::default(), writer_version, + write_path_in_schema, } } @@ -415,6 +419,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { pub struct ParquetMetaDataWriter<'a, W: Write> { buf: TrackedWrite, metadata: &'a ParquetMetaData, + write_path_in_schema: bool, } impl<'a, W: Write> ParquetMetaDataWriter<'a, W> { @@ -436,7 +441,20 @@ impl<'a, W: Write> ParquetMetaDataWriter<'a, W> { /// /// See example on the struct level documentation pub fn new_with_tracked(buf: TrackedWrite, metadata: &'a ParquetMetaData) -> Self { - Self { buf, metadata } + Self { + buf, + metadata, + write_path_in_schema: true, + } + } + + /// Set whether or not to write the `path_in_schema` field in the Thrift `ColumnMetaData` + /// struct. + pub fn with_write_path_in_schema(self, val: bool) -> Self { + Self { + write_path_in_schema: val, + ..self + } } /// Write the metadata to the buffer @@ -460,6 +478,7 @@ impl<'a, W: Write> ParquetMetaDataWriter<'a, W> { row_groups, created_by, file_metadata.version(), + self.write_path_in_schema, ); if let Some(column_indexes) = column_indexes { diff --git a/parquet/src/file/properties.rs b/parquet/src/file/properties.rs index 197c5d5c72a4..1ab8b5c710ba 100644 --- a/parquet/src/file/properties.rs +++ b/parquet/src/file/properties.rs @@ -69,6 +69,8 @@ pub const DEFAULT_OFFSET_INDEX_DISABLED: bool = false; pub const DEFAULT_COERCE_TYPES: bool = false; /// Default value for [`WriterProperties::data_page_v2_compression_ratio_threshold`] pub const DEFAULT_DATA_PAGE_V2_COMPRESSION_RATIO_THRESHOLD: f64 = 1.0; +/// Default value for [`WriterProperties::write_path_in_schema`] +pub const DEFAULT_WRITE_PATH_IN_SCHEMA: bool = true; /// Default minimum chunk size for content-defined chunking: 256 KiB. pub const DEFAULT_CDC_MIN_CHUNK_SIZE: usize = 256 * 1024; /// Default maximum chunk size for content-defined chunking: 1024 KiB. @@ -252,6 +254,7 @@ pub struct WriterProperties { statistics_truncate_length: Option, coerce_types: bool, content_defined_chunking: Option, + write_path_in_schema: bool, #[cfg(feature = "encryption")] pub(crate) file_encryption_properties: Option>, } @@ -437,6 +440,14 @@ impl WriterProperties { self.coerce_types } + /// Returns `true` if the `path_in_schema` field of the `ColumnMetaData` Thrift struct + /// should be written. + /// + /// For more details see [`WriterPropertiesBuilder::set_write_path_in_schema`] + pub fn write_path_in_schema(&self) -> bool { + self.write_path_in_schema + } + /// EXPERIMENTAL: Returns content-defined chunking options, or `None` if CDC is disabled. /// /// For more details see [`WriterPropertiesBuilder::set_content_defined_chunking`] @@ -592,6 +603,7 @@ pub struct WriterPropertiesBuilder { statistics_truncate_length: Option, coerce_types: bool, content_defined_chunking: Option, + write_path_in_schema: bool, #[cfg(feature = "encryption")] file_encryption_properties: Option>, } @@ -616,6 +628,7 @@ impl Default for WriterPropertiesBuilder { statistics_truncate_length: DEFAULT_STATISTICS_TRUNCATE_LENGTH, coerce_types: DEFAULT_COERCE_TYPES, content_defined_chunking: None, + write_path_in_schema: DEFAULT_WRITE_PATH_IN_SCHEMA, #[cfg(feature = "encryption")] file_encryption_properties: None, } @@ -670,6 +683,7 @@ impl WriterPropertiesBuilder { statistics_truncate_length: self.statistics_truncate_length, coerce_types: self.coerce_types, content_defined_chunking: self.content_defined_chunking, + write_path_in_schema: self.write_path_in_schema, #[cfg(feature = "encryption")] file_encryption_properties: self.file_encryption_properties, } @@ -885,6 +899,43 @@ impl WriterPropertiesBuilder { self } + /// EXPERIMENTAL: Should the writer emit the `path_in_schema` element of the + /// `ColumnMetaData` Thrift struct. Defaults to `true` via [`DEFAULT_WRITE_PATH_IN_SCHEMA`]. + /// + /// Because `path_in_schema` is a field on the `ColumnMetaData`, it is repeated + /// `num_columns * num_rowgroups` times. Compounding this is any level of nesting or + /// repetition in the schema. For instance, a top-level list column named `foo` will have + /// a `path_in_schema` of `["foo", "list", "element"]`. A list-of-struct is even worse, + /// because the necessary list wrapping is repeated for each element of the struct. A + /// file with a deeply nested schema and many row groups can have a large percentage of the + /// footer taken up by this field. For example, a file of 38 row groups with a schema containing + /// several lists of structs containing lists had 36% of the footer taken up by `path_in_schema`. + /// Removing this redundant information can greatly speed up footer parsing, which is particularly + /// important in scenarios where one does not wish to read the entire file (e.g. point + /// lookups). + /// + ///
+ /// + /// **WARNING:** + /// Setting this to `false` will break compatibility with Parquet readers that + /// still expect this field to be present. Virtually all Parquet readers (parquet-java, + /// Spark, arrow-cpp, pyarrow, pandas to name a few), with the exception + /// of the one in this crate, expect this field to be present, and will terminate execution + /// if it is not. This will continue to be the case unless/until the Parquet format + /// specification is explicitly changed to allow this field to be missing. As a consquence, + /// users should only set this to `false` if they have verified that any reader(s) they plan + /// to use can tolerate the absence of this field. + /// + /// For more context, see [GH-563]. + /// + ///
+ /// + /// [GH-563]: https://github.com/apache/parquet-format/issues/563 + pub fn set_write_path_in_schema(mut self, write_path_in_schema: bool) -> Self { + self.write_path_in_schema = write_path_in_schema; + self + } + /// EXPERIMENTAL: Sets content-defined chunking options, or disables CDC with `None`. /// /// When enabled, data page boundaries are determined by a rolling hash of the @@ -1253,6 +1304,7 @@ impl From for WriterPropertiesBuilder { statistics_truncate_length: props.statistics_truncate_length, coerce_types: props.coerce_types, content_defined_chunking: props.content_defined_chunking, + write_path_in_schema: props.write_path_in_schema, #[cfg(feature = "encryption")] file_encryption_properties: props.file_encryption_properties, } diff --git a/parquet/src/file/writer.rs b/parquet/src/file/writer.rs index 7d69904451d3..beb83b71cebc 100644 --- a/parquet/src/file/writer.rs +++ b/parquet/src/file/writer.rs @@ -345,12 +345,14 @@ impl SerializedFileWriter { let column_indexes = std::mem::take(&mut self.column_indexes); let offset_indexes = std::mem::take(&mut self.offset_indexes); + let write_path_in_schema = self.props.write_path_in_schema(); let mut encoder = ThriftMetadataWriter::new( &mut self.buf, &self.descr, row_groups, Some(self.props.created_by().to_string()), self.props.writer_version().as_num(), + write_path_in_schema, ); #[cfg(feature = "encryption")] diff --git a/parquet/src/parquet_thrift.rs b/parquet/src/parquet_thrift.rs index a13baa0924e0..e621f4c4989b 100644 --- a/parquet/src/parquet_thrift.rs +++ b/parquet/src/parquet_thrift.rs @@ -726,12 +726,29 @@ where /// [compact output]: https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md pub(crate) struct ThriftCompactOutputProtocol { writer: W, + write_path_in_schema: bool, } impl ThriftCompactOutputProtocol { /// Create a new `ThriftCompactOutputProtocol` wrapping the byte sink `writer`. pub(crate) fn new(writer: W) -> Self { - Self { writer } + Self { + writer, + write_path_in_schema: true, + } + } + + // TODO(ets): at some point there should probably be a properties object + // to control aspects of thrift output. But since this is the only option to date + // I'm choosing a simpler API. + /// Control the writing of the `path_in_schema` element of the `ColumnMetaData` + pub(crate) fn set_write_path_in_schema(&mut self, val: bool) { + self.write_path_in_schema = val; + } + + /// Indicate whether or not to emit `path_in_schema`. + pub(crate) fn write_path_in_schema(&self) -> bool { + self.write_path_in_schema } /// Write a single byte to the output stream. From e45354a4137ce92380953dcc23ffbf909cfa3539 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Thu, 7 May 2026 18:56:13 +0100 Subject: [PATCH 22/33] Remove deprecated legacy `like` kernels in `arrow-string` (#9674) # Which issue does this PR close? - Closes #9675 # Rationale for this change The legacy kernels have been deprecated for 2 years and hidden from docs, and its just more code to build. # What changes are included in this PR? 1. Removing long deprecated legacy like kernels 2. Keeps all tests, just folded them into the existing pattern. This also increases the effective coverage - testing scalar comparison both for Utf8 scalars and Dict scalars # Are these changes tested? No functional changes, but refactors existing tests # Are there any user-facing changes? Only for users using long hidden and deprecated functions, but I this isn't a meaningfully public API. The functionality is also still available with very minor changes. Co-authored-by: Andrew Lamb --- Cargo.lock | 52 +-- arrow-string/src/like.rs | 752 ++++----------------------------------- 2 files changed, 98 insertions(+), 706 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a3465ba9e458..11979eb5836b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -170,7 +170,7 @@ dependencies = [ [[package]] name = "arrow" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-arith", "arrow-array", @@ -197,7 +197,7 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-array", "arrow-buffer", @@ -209,7 +209,7 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.2.0" +version = "58.3.0" dependencies = [ "ahash", "arrow-buffer", @@ -229,7 +229,7 @@ dependencies = [ [[package]] name = "arrow-avro" -version = "58.2.0" +version = "58.3.0" dependencies = [ "apache-avro", "arrow", @@ -266,7 +266,7 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.2.0" +version = "58.3.0" dependencies = [ "bytes", "criterion", @@ -278,7 +278,7 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-array", "arrow-buffer", @@ -301,7 +301,7 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-array", "arrow-buffer", @@ -319,7 +319,7 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-buffer", "arrow-schema", @@ -330,7 +330,7 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "58.2.0" +version = "58.3.0" dependencies = [ "anyhow", "arrow-arith", @@ -370,7 +370,7 @@ dependencies = [ [[package]] name = "arrow-integration-test" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow", "arrow-buffer", @@ -383,7 +383,7 @@ dependencies = [ [[package]] name = "arrow-integration-testing" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow", "arrow-buffer", @@ -402,7 +402,7 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-array", "arrow-buffer", @@ -421,7 +421,7 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arbitrary", "arrow-array", @@ -452,7 +452,7 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-array", "arrow-buffer", @@ -465,7 +465,7 @@ dependencies = [ [[package]] name = "arrow-pyarrow" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-array", "arrow-data", @@ -475,7 +475,7 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-array", "arrow-buffer", @@ -490,7 +490,7 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.2.0" +version = "58.3.0" dependencies = [ "bitflags", "criterion", @@ -503,7 +503,7 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.2.0" +version = "58.3.0" dependencies = [ "ahash", "arrow-array", @@ -516,7 +516,7 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-array", "arrow-buffer", @@ -2327,7 +2327,7 @@ dependencies = [ [[package]] name = "parquet" -version = "58.2.0" +version = "58.3.0" dependencies = [ "ahash", "arrow", @@ -2378,7 +2378,7 @@ dependencies = [ [[package]] name = "parquet-geospatial" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-schema", "geo-traits", @@ -2390,7 +2390,7 @@ dependencies = [ [[package]] name = "parquet-variant" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow", "arrow-schema", @@ -2407,7 +2407,7 @@ dependencies = [ [[package]] name = "parquet-variant-compute" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow", "arrow-schema", @@ -2424,7 +2424,7 @@ dependencies = [ [[package]] name = "parquet-variant-json" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-schema", "base64", @@ -2436,7 +2436,7 @@ dependencies = [ [[package]] name = "parquet_derive" -version = "58.2.0" +version = "58.3.0" dependencies = [ "parquet", "proc-macro2", @@ -2446,7 +2446,7 @@ dependencies = [ [[package]] name = "parquet_derive_test" -version = "58.2.0" +version = "58.3.0" dependencies = [ "chrono", "parquet", diff --git a/arrow-string/src/like.rs b/arrow-string/src/like.rs index 2ad7e1cfd961..e9077d4b090d 100644 --- a/arrow-string/src/like.rs +++ b/arrow-string/src/like.rs @@ -28,8 +28,6 @@ use arrow_array::*; use arrow_schema::*; use arrow_select::take::take; -use std::sync::Arc; - use crate::binary_like::binary_apply; pub use arrow_array::StringArrayType; @@ -433,118 +431,7 @@ fn binary_predicate<'a>( .collect() } -// Deprecated kernels - -fn make_scalar(data_type: &DataType, scalar: &str) -> Result { - match data_type { - DataType::Utf8 => Ok(Arc::new(StringArray::from_iter_values([scalar]))), - DataType::LargeUtf8 => Ok(Arc::new(LargeStringArray::from_iter_values([scalar]))), - DataType::Dictionary(_, v) => make_scalar(v.as_ref(), scalar), - d => Err(ArrowError::InvalidArgumentError(format!( - "Unsupported string scalar data type {d:?}", - ))), - } -} - -macro_rules! legacy_kernels { - ($fn_datum:ident, $fn_array:ident, $fn_scalar:ident, $fn_array_dyn:ident, $fn_scalar_dyn:ident, $deprecation:expr) => { - #[doc(hidden)] - #[deprecated(note = $deprecation)] - pub fn $fn_array( - left: &GenericStringArray, - right: &GenericStringArray, - ) -> Result { - $fn_datum(left, right) - } - - #[doc(hidden)] - #[deprecated(note = $deprecation)] - pub fn $fn_scalar( - left: &GenericStringArray, - right: &str, - ) -> Result { - let scalar = GenericStringArray::::from_iter_values([right]); - $fn_datum(left, &Scalar::new(&scalar)) - } - - #[doc(hidden)] - #[deprecated(note = $deprecation)] - pub fn $fn_array_dyn( - left: &dyn Array, - right: &dyn Array, - ) -> Result { - $fn_datum(&left, &right) - } - - #[doc(hidden)] - #[deprecated(note = $deprecation)] - pub fn $fn_scalar_dyn(left: &dyn Array, right: &str) -> Result { - let scalar = make_scalar(left.data_type(), right)?; - $fn_datum(&left, &Scalar::new(&scalar)) - } - }; -} - -legacy_kernels!( - like, - like_utf8, - like_utf8_scalar, - like_dyn, - like_utf8_scalar_dyn, - "Use arrow_string::like::like" -); -legacy_kernels!( - ilike, - ilike_utf8, - ilike_utf8_scalar, - ilike_dyn, - ilike_utf8_scalar_dyn, - "Use arrow_string::like::ilike" -); -legacy_kernels!( - nlike, - nlike_utf8, - nlike_utf8_scalar, - nlike_dyn, - nlike_utf8_scalar_dyn, - "Use arrow_string::like::nlike" -); -legacy_kernels!( - nilike, - nilike_utf8, - nilike_utf8_scalar, - nilike_dyn, - nilike_utf8_scalar_dyn, - "Use arrow_string::like::nilike" -); -legacy_kernels!( - contains, - contains_utf8, - contains_utf8_scalar, - contains_dyn, - contains_utf8_scalar_dyn, - "Use arrow_string::like::contains" -); -legacy_kernels!( - starts_with, - starts_with_utf8, - starts_with_utf8_scalar, - starts_with_dyn, - starts_with_utf8_scalar_dyn, - "Use arrow_string::like::starts_with" -); - -legacy_kernels!( - ends_with, - ends_with_utf8, - ends_with_utf8_scalar, - ends_with_dyn, - ends_with_utf8_scalar_dyn, - "Use arrow_string::like::ends_with" -); - #[cfg(test)] -#[allow(deprecated)] mod tests { use super::*; use arrow_array::builder::BinaryDictionaryBuilder; @@ -688,6 +575,10 @@ mod tests { assert_eq!(res, expected); let left: DictionaryArray = $left.into_iter().collect(); + let right = StringArray::from_iter_values([$right]); + let res = $op(&left, &Scalar::new(&right)).unwrap(); + assert_eq!(res, expected); + let right: DictionaryArray = [$right].into_iter().collect(); let res = $op(&left, &Scalar::new(&right)).unwrap(); assert_eq!(res, expected); @@ -1429,601 +1320,102 @@ mod tests { vec![true, false, true, true, true] ); - test_utf8!( - test_utf8_array_eq_ignore_ascii_case, - vec!["arrow", "arrow", "arrow", "arrow", "parquet", "parquet"], - vec!["arrow", "ARROW", "arro", "aRrOw", "arrow", "ARROW"], - eq_ignore_ascii_case, - vec![true, true, false, true, false, false] - ); - + // Nullable, repeated values exercise dictionary remapping with a plain UTF8 scalar RHS. test_utf8_scalar!( - test_utf8_array_eq_ignore_ascii_case_scalar, - vec!["arrow", "aRrOW", "arro", "ARROW", "parquet", "PARQUET"], - "arrow", - eq_ignore_ascii_case, - vec![true, true, false, true, false, false] - ); - - #[test] - fn test_dict_like_kernels() { - let data = vec![ + test_utf8_scalar_nullable_like, + vec![ Some("Earth"), Some("Fire"), Some("Water"), Some("Air"), None, Some("Air"), - Some("bbbbb\nAir"), - ]; - - let dict_array: DictionaryArray = data.into_iter().collect(); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "Air").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(false), - Some(true), - None, - Some(true), - Some(false), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "Air").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(false), - Some(true), - None, - Some(true), - Some(false), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "Wa%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "Wa%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "%r").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "%r").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "%i%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "%i%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "%a%r%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "%a%r%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - } + Some("bbbbb\nAir") + ], + "Air", + like, + vec![ + Some(false), + Some(false), + Some(false), + Some(true), + None, + Some(true), + Some(false) + ] + ); - #[test] - fn test_dict_nlike_kernels() { - let data = vec![ + test_utf8_scalar!( + test_utf8_scalar_nullable_nlike, + vec![ Some("Earth"), Some("Fire"), Some("Water"), Some("Air"), None, Some("Air"), - Some("bbbbb\nAir"), - ]; - - let dict_array: DictionaryArray = data.into_iter().collect(); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "Air").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(true), - Some(false), - None, - Some(false), - Some(true), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "Air").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(true), - Some(false), - None, - Some(false), - Some(true), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "Wa%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "Wa%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "%r").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "%r").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "%i%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "%i%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "%a%r%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "%a%r%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - } + Some("bbbbb\nAir") + ], + "%a%r%", + nlike, + vec![ + Some(false), + Some(true), + Some(false), + Some(true), + None, + Some(true), + Some(true) + ] + ); - #[test] - fn test_dict_ilike_kernels() { - let data = vec![ + test_utf8_scalar!( + test_utf8_scalar_nullable_ilike, + vec![ Some("Earth"), Some("Fire"), Some("Water"), Some("Air"), None, Some("Air"), - Some("bbbbb\nAir"), - ]; - - let dict_array: DictionaryArray = data.into_iter().collect(); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "air").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(false), - Some(true), - None, - Some(true), - Some(false), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "air").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(false), - Some(true), - None, - Some(true), - Some(false), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "wa%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "wa%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "%R").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "%R").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "%I%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "%I%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "%A%r%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "%A%r%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - } + Some("bbbbb\nAir") + ], + "%I%", + ilike, + vec![ + Some(false), + Some(true), + Some(false), + Some(true), + None, + Some(true), + Some(true) + ] + ); - #[test] - fn test_dict_nilike_kernels() { - let data = vec![ + test_utf8_scalar!( + test_utf8_scalar_nullable_nilike, + vec![ Some("Earth"), Some("Fire"), Some("Water"), Some("Air"), None, Some("Air"), - Some("bbbbb\nAir"), - ]; - - let dict_array: DictionaryArray = data.into_iter().collect(); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "air").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(true), - Some(false), - None, - Some(false), - Some(true), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "air").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(true), - Some(false), - None, - Some(false), - Some(true), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "wa%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "wa%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "%R").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "%R").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "%I%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "%I%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "%A%r%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "%A%r%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - } + Some("bbbbb\nAir") + ], + "%R", + nilike, + vec![ + Some(true), + Some(true), + Some(false), + Some(false), + None, + Some(false), + Some(false) + ] + ); #[test] fn string_null_like_pattern() { From 76c381ff0a2cd3f0b5f6690b3491428ffa0806a0 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 7 May 2026 13:56:33 -0400 Subject: [PATCH 23/33] Remove redundant benchmarks in `cast_kernels` (#9789) # Which issue does this PR close? - Follow on to https://github.com/apache/arrow-rs/pull/9729 # Rationale for this change #9729, added many new cases to `cast_kernels` but many of these are redundant and increase the benchmark runtime without providing proportional value in coverage. This PR reduces the redundancy by: 1. Keeps one representative benchmark for each major physical code path (e.g., `i128` vs `i256` storage). 2. Removes redundant combinations of target types (e.g., casting `decimal128` to every integer width when `int64` is sufficient). 3. Consolidating invalid/error path testing into a single representative case. 4. Reducing the total number of benchmark cases from over 60 new additions to 10 high-value cases. # What changes are included in this PR? - Pruned redundant decimal-to-integer/float and string/float-to-decimal benchmarks in `arrow/benches/cast_kernels.rs`. - Added `create_primitive_array_range` helper to `arrow/src/util/bench_util.rs` Compared to main before PR #9729, the following benchmarks will be new after my PR #9789 is merged: 1. New Decimal Casting Benchmarks These cases cover the core performance paths for casting to and from decimals using representative physical storage types (i128 and i256): * cast string to decimal128(38, 3) * cast float64 to decimal128(32, 3) * cast invalid float64 to to decimal128(32, 3) (Error path testing) * cast decimal128 to float64 * cast decimal128 to int64 * cast decimal256 to float64 * cast decimal256 to int64 * cast decimal128 to decimal128 512 with lower scale (infallible) (specifically testing the fast path for infallible # Are these changes tested? CI covers verification. # Are there any user-facing changes? No. --- arrow/benches/cast_kernels.rs | 343 +--------------------------------- 1 file changed, 8 insertions(+), 335 deletions(-) diff --git a/arrow/benches/cast_kernels.rs b/arrow/benches/cast_kernels.rs index 2af3145a84e1..82e9696f0458 100644 --- a/arrow/benches/cast_kernels.rs +++ b/arrow/benches/cast_kernels.rs @@ -83,36 +83,6 @@ fn build_utf8_date_time_array(size: usize, with_nulls: bool) -> ArrayRef { Arc::new(builder.finish()) } -fn build_decimal32_array(size: usize, precision: u8, scale: i8) -> ArrayRef { - let mut rng = seedable_rng(); - let mut builder = Decimal32Builder::with_capacity(size); - - for _ in 0..size { - builder.append_value(rng.random_range::(0..1000000)); - } - Arc::new( - builder - .finish() - .with_precision_and_scale(precision, scale) - .unwrap(), - ) -} - -fn build_decimal64_array(size: usize, precision: u8, scale: i8) -> ArrayRef { - let mut rng = seedable_rng(); - let mut builder = Decimal64Builder::with_capacity(size); - - for _ in 0..size { - builder.append_value(rng.random_range::(0..1000000000)); - } - Arc::new( - builder - .finish() - .with_precision_and_scale(precision, scale) - .unwrap(), - ) -} - fn build_decimal128_array(size: usize, precision: u8, scale: i8) -> ArrayRef { let mut rng = seedable_rng(); let mut builder = Decimal128Builder::with_capacity(size); @@ -175,6 +145,14 @@ fn build_string_float_array(size: usize, null_density: f32) -> ArrayRef { Arc::new(builder.finish()) } +fn build_float64_array_for_cast_to_decimal(size: usize, null_density: f32) -> ArrayRef { + Arc::new(create_primitive_array_range::( + size, + null_density, + -999_999_999f64..999_999_999f64, + )) +} + macro_rules! build_array_with_samples { ($builder: ident, $size: ident, $null_density: expr, $samples: ident) => {{ let mut rng = seedable_rng(); @@ -189,46 +167,6 @@ macro_rules! build_array_with_samples { }}; } -fn build_string_float_array_invalid_item(size: usize, null_density: f32) -> ArrayRef { - let invalid_items = [ - "--1.23", - "1.2.3", - "-1.-23499", - "--1.23456789", - "1-.234", - "e10", - "1e", - "1e++10", - "NaN", - "Infinity", - ]; - let mut builder = StringBuilder::new(); - build_array_with_samples!(builder, size, null_density, invalid_items) -} - -fn build_float32_array_for_cast_to_decimal(size: usize, null_density: f32) -> ArrayRef { - Arc::new(create_primitive_array_range::( - size, - null_density, - -999_999_999f32..999_999_999f32, - )) -} - -fn build_float64_array_for_cast_to_decimal(size: usize, null_density: f32) -> ArrayRef { - Arc::new(create_primitive_array_range::( - size, - null_density, - -999_999_999f64..999_999_999f64, - )) -} - -fn build_float32_array_invalid_item(size: usize, null_density: f32) -> ArrayRef { - let mut builder = Float32Builder::with_capacity(size); - let invalid_values = [f32::NAN, f32::INFINITY, f32::NEG_INFINITY]; - - build_array_with_samples!(builder, size, null_density, invalid_values) -} - fn build_float64_array_invalid_items(size: usize, null_density: f32) -> ArrayRef { let mut builder = Float64Builder::with_capacity(size); let invalid_values = [f64::NAN, f64::INFINITY, f64::NEG_INFINITY]; @@ -268,8 +206,6 @@ fn add_benchmark(c: &mut Criterion) { let utf8_date_array = build_utf8_date_array(512, true); let utf8_date_time_array = build_utf8_date_time_array(512, true); - let decimal32_array = build_decimal32_array(8_000, 9, 3); - let decimal64_array = build_decimal64_array(8_000, 10, 3); let decimal128_array = build_decimal128_array(8_000, 10, 3); let decimal256_array = build_decimal256_array(8_000, 50, 3); let string_array = build_string_array(512); @@ -280,10 +216,7 @@ fn add_benchmark(c: &mut Criterion) { let binary_view_array = cast(&string_view_array, &DataType::BinaryView).unwrap(); let string_float_array_normal = build_string_float_array(5_000, 0.1); - let invalid_string_float_array = build_string_float_array_invalid_item(8_000, 0.1); - let float32_array_cast_to_decimal = build_float32_array_for_cast_to_decimal(8_000, 0.1); let float64_array_cast_to_decimal = build_float64_array_for_cast_to_decimal(8_000, 0.1); - let invalid_float32_array_to_decimal = build_float32_array_invalid_item(8_000, 0.1); let invalid_float64_array_to_decimal = build_float64_array_invalid_items(8_000, 0.1); c.bench_function("cast int32 to int32 512", |b| { @@ -366,22 +299,6 @@ fn add_benchmark(c: &mut Criterion) { b.iter(|| cast_array(&utf8_date_time_array, DataType::Date64)) }); - c.bench_function("cast decimal32 to decimal32 512", |b| { - b.iter(|| cast_array(&decimal32_array, DataType::Decimal32(9, 4))) - }); - c.bench_function("cast decimal32 to decimal32 512 lower precision", |b| { - b.iter(|| cast_array(&decimal32_array, DataType::Decimal32(6, 5))) - }); - c.bench_function("cast decimal32 to decimal64 512", |b| { - b.iter(|| cast_array(&decimal32_array, DataType::Decimal64(11, 5))) - }); - c.bench_function("cast decimal64 to decimal32 512", |b| { - b.iter(|| cast_array(&decimal64_array, DataType::Decimal32(9, 2))) - }); - c.bench_function("cast decimal64 to decimal64 512", |b| { - b.iter(|| cast_array(&decimal64_array, DataType::Decimal64(12, 4))) - }); - c.bench_function("cast decimal128 to decimal128 512", |b| { b.iter(|| cast_array(&decimal128_array, DataType::Decimal128(30, 5))) }); @@ -455,288 +372,44 @@ fn add_benchmark(c: &mut Criterion) { } // cast string with normal items to decimals - benchmark_cast!( - "cast string to decimal32(9, 2)", - string_float_array_normal, - DataType::Decimal32(9, 2) - ); - benchmark_cast!( - "cast string to decimal64(18, 2)", - string_float_array_normal, - DataType::Decimal64(18, 2) - ); benchmark_cast!( "cast string to decimal128(38, 3)", string_float_array_normal, DataType::Decimal128(38, 3) ); - benchmark_cast!( - "cast string to decimal256(76, 4)", - string_float_array_normal, - DataType::Decimal256(76, 4) - ); - - // cast invalid string to decimals - benchmark_cast!( - "cast invalid string to decimal32(9, 2)", - invalid_string_float_array, - DataType::Decimal32(9, 2) - ); - benchmark_cast!( - "cast invalid string to decimal64(18, 2)", - invalid_string_float_array, - DataType::Decimal64(18, 2) - ); - benchmark_cast!( - "cast invalid string to decimal128(38, 3)", - invalid_string_float_array, - DataType::Decimal128(38, 3) - ); - benchmark_cast!( - "cast invalid string to decimal256(76, 4)", - invalid_string_float_array, - DataType::Decimal256(76, 4) - ); - - // cast float32 to decimals - benchmark_cast!( - "cast float32 to decimal32(9, 2)", - float32_array_cast_to_decimal, - DataType::Decimal32(9, 2) - ); - benchmark_cast!( - "cast float32 to decimal64(18, 2", - float32_array_cast_to_decimal, - DataType::Decimal64(18, 2) - ); - benchmark_cast!( - "cast float32 to decimal128(32, 3)", - float32_array_cast_to_decimal, - DataType::Decimal128(38, 3) - ); - benchmark_cast!( - "cast float32 to decimal256(76, 4)", - float32_array_cast_to_decimal, - DataType::Decimal256(76, 4) - ); - - // cast invalid float32 to decimals - benchmark_cast!( - "cast invalid float32 to decimal32(9, 2)", - invalid_float32_array_to_decimal, - DataType::Decimal32(9, 2) - ); - benchmark_cast!( - "cast invalid float32 to decimal64(18, 2", - invalid_float32_array_to_decimal, - DataType::Decimal64(18, 2) - ); - benchmark_cast!( - "cast invalid float32 to decimal128(32, 3)", - invalid_float32_array_to_decimal, - DataType::Decimal128(32, 3) - ); - benchmark_cast!( - "cast invalid float32 to decimal256(76, 4)", - invalid_float32_array_to_decimal, - DataType::Decimal256(76, 4) - ); // cast float64 to decimals - benchmark_cast!( - "cast float64 to decimal32(9, 2)", - float64_array_cast_to_decimal, - DataType::Decimal32(9, 2) - ); - benchmark_cast!( - "cast float64 to decimal64(18, 2", - float64_array_cast_to_decimal, - DataType::Decimal64(18, 2) - ); benchmark_cast!( "cast float64 to decimal128(32, 3)", float64_array_cast_to_decimal, DataType::Decimal128(32, 3) ); - benchmark_cast!( - "cast float64 to decimal256(76, 4)", - float64_array_cast_to_decimal, - DataType::Decimal256(76, 4) - ); // cast invalid float64 to decimals - benchmark_cast!( - "cast invalid float64 to decimal32(9, 2)", - invalid_float64_array_to_decimal, - DataType::Decimal32(9, 2) - ); - benchmark_cast!( - "cast invalid float64 to to decimal64(18, 2)", - invalid_float64_array_to_decimal, - DataType::Decimal64(18, 2) - ); benchmark_cast!( "cast invalid float64 to to decimal128(32, 3)", invalid_float64_array_to_decimal, DataType::Decimal128(32, 3) ); - benchmark_cast!( - "cast invalid float64 to to decimal256(76, 4)", - invalid_float64_array_to_decimal, - DataType::Decimal256(76, 4) - ); // cast decimals to float/integers - benchmark_cast!( - "cast decimal32 to float32", - decimal32_array, - DataType::Float32 - ); - benchmark_cast!( - "cast decimal32 to float64", - decimal32_array, - DataType::Float64 - ); - benchmark_cast!("cast decimal32 to uint8", decimal32_array, DataType::UInt8); - benchmark_cast!( - "cast decimal32 to uint16", - decimal32_array, - DataType::UInt16 - ); - benchmark_cast!( - "cast decimal32 to uint32", - decimal32_array, - DataType::UInt32 - ); - benchmark_cast!( - "cast decimal32 to uint64", - decimal32_array, - DataType::UInt64 - ); - benchmark_cast!("cast decimal32 to int8", decimal32_array, DataType::Int8); - benchmark_cast!("cast decimal32 to int16", decimal32_array, DataType::Int16); - benchmark_cast!("cast decimal32 to int32", decimal32_array, DataType::Int32); - benchmark_cast!("cast decimal32 to int64", decimal32_array, DataType::Int64); - - benchmark_cast!( - "cast decimal64 to float32", - decimal64_array, - DataType::Float32 - ); - benchmark_cast!( - "cast decimal64 to float64", - decimal64_array, - DataType::Float64 - ); - benchmark_cast!("cast decimal64 to uint8", decimal64_array, DataType::UInt8); - benchmark_cast!( - "cast decimal64 to uint16", - decimal64_array, - DataType::UInt16 - ); - benchmark_cast!( - "cast decimal64 to uint32", - decimal64_array, - DataType::UInt32 - ); - benchmark_cast!( - "cast decimal64 to uint64", - decimal64_array, - DataType::UInt64 - ); - benchmark_cast!("cast decimal64 to int8", decimal64_array, DataType::Int8); - benchmark_cast!("cast decimal64 to int16", decimal64_array, DataType::Int16); - benchmark_cast!("cast decimal64 to int32", decimal64_array, DataType::Int32); - benchmark_cast!("cast decimal64 to int64", decimal64_array, DataType::Int64); - - benchmark_cast!( - "cast decimal128 to float32", - decimal128_array, - DataType::Float32 - ); benchmark_cast!( "cast decimal128 to float64", decimal128_array, DataType::Float64 ); - benchmark_cast!( - "cast decimal128 to uint8", - decimal128_array, - DataType::UInt8 - ); - benchmark_cast!( - "cast decimal128 to uint16", - decimal128_array, - DataType::UInt16 - ); - benchmark_cast!( - "cast decimal128 to uint32", - decimal128_array, - DataType::UInt32 - ); - benchmark_cast!( - "cast decimal128 to uint64", - decimal128_array, - DataType::UInt64 - ); benchmark_cast!("cast decimal128 to int8", decimal128_array, DataType::Int8); - benchmark_cast!( - "cast decimal128 to int16", - decimal128_array, - DataType::Int16 - ); - benchmark_cast!( - "cast decimal128 to int32", - decimal128_array, - DataType::Int32 - ); benchmark_cast!( "cast decimal128 to int64", decimal128_array, DataType::Int64 ); - benchmark_cast!( - "cast decimal256 to float32", - decimal256_array, - DataType::Float32 - ); benchmark_cast!( "cast decimal256 to float64", decimal256_array, DataType::Float64 ); - benchmark_cast!( - "cast decimal256 to uint8", - decimal256_array, - DataType::UInt8 - ); - benchmark_cast!( - "cast decimal256 to uint16", - decimal256_array, - DataType::UInt16 - ); - benchmark_cast!( - "cast decimal256 to uint32", - decimal256_array, - DataType::UInt32 - ); - benchmark_cast!( - "cast decimal256 to uint64", - decimal256_array, - DataType::UInt64 - ); - benchmark_cast!("cast decimal256 to int8", decimal256_array, DataType::Int8); - benchmark_cast!( - "cast decimal256 to int16", - decimal256_array, - DataType::Int16 - ); - benchmark_cast!( - "cast decimal256 to int32", - decimal256_array, - DataType::Int32 - ); benchmark_cast!( "cast decimal256 to int64", decimal256_array, From 5d464b58f9fc183141e4501df6759e17d4c71507 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Thu, 7 May 2026 10:56:58 -0700 Subject: [PATCH 24/33] Add `CompressionCodec` Thrift enum for Parquet metadata (#9864) # Which issue does this PR close? - Part of #9863. # Rationale for this change See issue for more, but the idea is to separate Parquet metadata structures from those used for configuration. This can reduce memory used by the metadata, and also allows use of the thrift macros, reducing maintenance burden. # What changes are included in this PR? This adds a new `CompressionCodec` enum for use in the Parquet metadata, and means to convert between `CompressionCodec` and `Compression`. # Are these changes tested? Should be covered by existing tests, but new test of the interchange is also added. # Are there any user-facing changes? No --- parquet/benches/metadata.rs | 2 +- parquet/src/basic.rs | 155 +++++++++++++++++------- parquet/src/bin/parquet-layout.rs | 22 ++-- parquet/src/file/metadata/memory.rs | 4 +- parquet/src/file/metadata/mod.rs | 63 ++++++---- parquet/src/file/metadata/thrift/mod.rs | 6 +- parquet/src/file/writer.rs | 4 +- parquet/src/schema/printer.rs | 6 +- 8 files changed, 167 insertions(+), 95 deletions(-) diff --git a/parquet/benches/metadata.rs b/parquet/benches/metadata.rs index 750e33a2d364..b6e04580c141 100644 --- a/parquet/benches/metadata.rs +++ b/parquet/benches/metadata.rs @@ -90,7 +90,7 @@ fn encoded_meta(is_nullable: bool, has_lists: bool, write_path_in_schema: bool) .map(|j| { ColumnChunkMetaData::builder(column_desc_ptrs[j].clone()) .set_encodings(vec![Encoding::PLAIN, Encoding::RLE_DICTIONARY]) - .set_compression(parquet::basic::Compression::UNCOMPRESSED) + .set_compression_codec(parquet::basic::CompressionCodec::UNCOMPRESSED) .set_num_values(rng.random_range(1..1000000)) .set_total_compressed_size(rng.random_range(50000..5000000)) .set_data_page_offset(rng.random_range(4..2000000000)) diff --git a/parquet/src/basic.rs b/parquet/src/basic.rs index ba8ffc2e92c3..cc7a16eca063 100644 --- a/parquet/src/basic.rs +++ b/parquet/src/basic.rs @@ -798,6 +798,33 @@ fn i32_to_encoding(val: i32) -> Encoding { // ---------------------------------------------------------------------- // Mirrors thrift enum `CompressionCodec` +thrift_enum!( +/// Supported compression algorithms. +/// +/// Codecs added in format version X.Y can be read by readers based on X.Y and later. +/// Codec support may vary between readers based on the format version and +/// libraries available at runtime. +/// +/// See [Compression.md] for a detailed specification of these algorithms. +/// +/// [Compression.md]: https://github.com/apache/parquet-format/blob/master/Compression.md +enum CompressionCodec { + UNCOMPRESSED = 0; + SNAPPY = 1; + GZIP = 2; + LZO = 3; + BROTLI = 4; // Added in 2.4 + LZ4 = 5; // DEPRECATED (Added in 2.4) + ZSTD = 6; // Added in 2.4 + LZ4_RAW = 7; // Added in 2.9 +} +); + +// NOTE: This enum likely belongs in file::properties now, but moving it there would be a +// breaking API change, that's probably not worth the pain. If a new codec is added to the +// Parquet specification, or any other breaking changes are made to this enum, this can be +// revisited. + /// Supported block compression algorithms. /// /// Block compression can yield non-trivial improvements to storage efficiency at the expense @@ -834,51 +861,33 @@ pub enum Compression { LZ4_RAW, } -impl<'a, R: ThriftCompactInputProtocol<'a>> ReadThrift<'a, R> for Compression { - fn read_thrift(prot: &mut R) -> Result { - let val = prot.read_i32()?; - Ok(match val { - 0 => Self::UNCOMPRESSED, - 1 => Self::SNAPPY, - 2 => Self::GZIP(Default::default()), - 3 => Self::LZO, - 4 => Self::BROTLI(Default::default()), - 5 => Self::LZ4, - 6 => Self::ZSTD(Default::default()), - 7 => Self::LZ4_RAW, - _ => return Err(general_err!("Unexpected CompressionCodec {}", val)), - }) - } -} - -// TODO(ets): explore replacing this with a thrift_enum!(ThriftCompression) for the serialization -// and then provide `From` impls to convert back and forth. This is necessary due to the addition -// of compression level to some variants. -impl WriteThrift for Compression { - const ELEMENT_TYPE: ElementType = ElementType::I32; - - fn write_thrift(&self, writer: &mut ThriftCompactOutputProtocol) -> Result<()> { - let id: i32 = match *self { - Self::UNCOMPRESSED => 0, - Self::SNAPPY => 1, - Self::GZIP(_) => 2, - Self::LZO => 3, - Self::BROTLI(_) => 4, - Self::LZ4 => 5, - Self::ZSTD(_) => 6, - Self::LZ4_RAW => 7, - }; - writer.write_i32(id) +impl From for Compression { + fn from(value: CompressionCodec) -> Self { + match value { + CompressionCodec::UNCOMPRESSED => Compression::UNCOMPRESSED, + CompressionCodec::SNAPPY => Compression::SNAPPY, + CompressionCodec::GZIP => Compression::GZIP(Default::default()), + CompressionCodec::LZO => Compression::LZO, + CompressionCodec::BROTLI => Compression::BROTLI(Default::default()), + CompressionCodec::LZ4 => Compression::LZ4, + CompressionCodec::ZSTD => Compression::ZSTD(Default::default()), + CompressionCodec::LZ4_RAW => Compression::LZ4_RAW, + } } } -write_thrift_field!(Compression, FieldType::I32); - -impl Compression { - /// Returns the codec type of this compression setting as a string, without the compression - /// level. - pub(crate) fn codec_to_string(self) -> String { - format!("{self:?}").split('(').next().unwrap().to_owned() +impl From for CompressionCodec { + fn from(value: Compression) -> Self { + match value { + Compression::UNCOMPRESSED => CompressionCodec::UNCOMPRESSED, + Compression::SNAPPY => CompressionCodec::SNAPPY, + Compression::GZIP(_) => CompressionCodec::GZIP, + Compression::LZO => CompressionCodec::LZO, + Compression::BROTLI(_) => CompressionCodec::BROTLI, + Compression::LZ4 => CompressionCodec::LZ4, + Compression::ZSTD(_) => CompressionCodec::ZSTD, + Compression::LZ4_RAW => CompressionCodec::LZ4_RAW, + } } } @@ -2140,11 +2149,65 @@ mod tests { } #[test] - fn test_compression_codec_to_string() { - assert_eq!(Compression::UNCOMPRESSED.codec_to_string(), "UNCOMPRESSED"); + fn test_compression_conversion() { + assert_eq!( + CompressionCodec::from(Compression::UNCOMPRESSED), + CompressionCodec::UNCOMPRESSED + ); + assert_eq!( + CompressionCodec::from(Compression::SNAPPY), + CompressionCodec::SNAPPY + ); + assert_eq!( + CompressionCodec::from(Compression::GZIP(Default::default())), + CompressionCodec::GZIP + ); + assert_eq!( + CompressionCodec::from(Compression::LZO), + CompressionCodec::LZO + ); + assert_eq!( + CompressionCodec::from(Compression::BROTLI(Default::default())), + CompressionCodec::BROTLI + ); + assert_eq!( + CompressionCodec::from(Compression::LZ4), + CompressionCodec::LZ4 + ); + assert_eq!( + CompressionCodec::from(Compression::ZSTD(Default::default())), + CompressionCodec::ZSTD + ); + assert_eq!( + CompressionCodec::from(Compression::LZ4_RAW), + CompressionCodec::LZ4_RAW + ); + + assert_eq!( + Compression::from(CompressionCodec::UNCOMPRESSED), + Compression::UNCOMPRESSED + ); + assert_eq!( + Compression::from(CompressionCodec::SNAPPY), + Compression::SNAPPY + ); + assert_eq!( + Compression::from(CompressionCodec::GZIP), + Compression::GZIP(Default::default()) + ); + assert_eq!(Compression::from(CompressionCodec::LZO), Compression::LZO); + assert_eq!( + Compression::from(CompressionCodec::BROTLI), + Compression::BROTLI(Default::default()) + ); + assert_eq!(Compression::from(CompressionCodec::LZ4), Compression::LZ4); + assert_eq!( + Compression::from(CompressionCodec::ZSTD), + Compression::ZSTD(Default::default()) + ); assert_eq!( - Compression::ZSTD(ZstdLevel::default()).codec_to_string(), - "ZSTD" + Compression::from(CompressionCodec::LZ4_RAW), + Compression::LZ4_RAW ); } diff --git a/parquet/src/bin/parquet-layout.rs b/parquet/src/bin/parquet-layout.rs index 007f93517d96..735a9665139a 100644 --- a/parquet/src/bin/parquet-layout.rs +++ b/parquet/src/bin/parquet-layout.rs @@ -48,7 +48,7 @@ use parquet::file::metadata::ParquetMetaDataReader; use serde::Serialize; use thrift::protocol::TCompactInputProtocol; -use parquet::basic::Compression; +use parquet::basic::CompressionCodec; use parquet::errors::Result; use parquet::file::reader::ChunkReader; #[allow(deprecated)] @@ -118,7 +118,7 @@ fn do_layout(reader: &C) -> Result { .iter() .zip(schema.columns()) .map(|(column, column_schema)| { - let compression = compression(column.compression()); + let compression = compression(column.compression_codec()); let mut pages = vec![]; let mut start = column @@ -225,16 +225,16 @@ fn read_page_header(reader: &C, offset: u64) -> Result<(usize, P } /// Returns a string representation for a given compression -fn compression(compression: Compression) -> Option<&'static str> { +fn compression(compression: CompressionCodec) -> Option<&'static str> { match compression { - Compression::UNCOMPRESSED => None, - Compression::SNAPPY => Some("snappy"), - Compression::GZIP(_) => Some("gzip"), - Compression::LZO => Some("lzo"), - Compression::BROTLI(_) => Some("brotli"), - Compression::LZ4 => Some("lz4"), - Compression::ZSTD(_) => Some("zstd"), - Compression::LZ4_RAW => Some("lz4_raw"), + CompressionCodec::UNCOMPRESSED => None, + CompressionCodec::SNAPPY => Some("snappy"), + CompressionCodec::GZIP => Some("gzip"), + CompressionCodec::LZO => Some("lzo"), + CompressionCodec::BROTLI => Some("brotli"), + CompressionCodec::LZ4 => Some("lz4"), + CompressionCodec::ZSTD => Some("zstd"), + CompressionCodec::LZ4_RAW => Some("lz4_raw"), } } diff --git a/parquet/src/file/metadata/memory.rs b/parquet/src/file/metadata/memory.rs index 30c10e7f2293..8a5937c43c65 100644 --- a/parquet/src/file/metadata/memory.rs +++ b/parquet/src/file/metadata/memory.rs @@ -18,7 +18,7 @@ //! Memory calculations for [`ParquetMetadata::memory_size`] //! //! [`ParquetMetadata::memory_size`]: crate::file::metadata::ParquetMetaData::memory_size -use crate::basic::{BoundaryOrder, ColumnOrder, Compression, Encoding, PageType}; +use crate::basic::{BoundaryOrder, ColumnOrder, CompressionCodec, Encoding, PageType}; use crate::data_type::private::ParquetValueType; use crate::file::metadata::{ ColumnChunkMetaData, FileMetaData, KeyValue, PageEncodingStats, ParquetPageEncodingStats, @@ -206,7 +206,7 @@ impl HeapSize for SortingColumn { 0 // no heap allocations } } -impl HeapSize for Compression { +impl HeapSize for CompressionCodec { fn heap_size(&self) -> usize { 0 // no heap allocations } diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index 156385ddaa70..646438d2e979 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -95,9 +95,13 @@ pub(crate) mod reader; pub(crate) mod thrift; mod writer; -use crate::basic::{EncodingMask, PageType}; +use crate::basic::{ + BoundaryOrder, ColumnOrder, Compression, CompressionCodec, Encoding, EncodingMask, PageType, + Type, +}; #[cfg(feature = "encryption")] use crate::encryption::decrypt::FileDecryptor; +use crate::errors::{ParquetError, Result}; #[cfg(feature = "encryption")] use crate::file::column_crypto_metadata::ColumnCryptoMetaData; pub(crate) use crate::file::metadata::memory::HeapSize; @@ -107,22 +111,15 @@ use crate::file::page_index::column_index::{ByteArrayColumnIndex, PrimitiveColum use crate::file::page_index::{column_index::ColumnIndexMetaData, offset_index::PageLocation}; use crate::file::statistics::Statistics; use crate::geospatial::statistics as geo_statistics; +use crate::parquet_thrift::{ + ElementType, FieldType, ReadThrift, ThriftCompactInputProtocol, ThriftCompactOutputProtocol, + WriteThrift, WriteThriftField, +}; use crate::schema::types::{ ColumnDescPtr, ColumnDescriptor, ColumnPath, SchemaDescPtr, SchemaDescriptor, Type as SchemaType, }; use crate::thrift_struct; -use crate::{ - basic::BoundaryOrder, - errors::{ParquetError, Result}, -}; -use crate::{ - basic::{ColumnOrder, Compression, Encoding, Type}, - parquet_thrift::{ - ElementType, FieldType, ReadThrift, ThriftCompactInputProtocol, - ThriftCompactOutputProtocol, WriteThrift, WriteThriftField, - }, -}; use crate::{ data_type::private::ParquetValueType, file::page_index::offset_index::OffsetIndexMetaData, }; @@ -814,7 +811,7 @@ pub struct ColumnChunkMetaData { file_path: Option, file_offset: i64, num_values: i64, - compression: Compression, + compression: CompressionCodec, total_compressed_size: i64, total_uncompressed_size: i64, data_page_offset: i64, @@ -1016,8 +1013,18 @@ impl ColumnChunkMetaData { self.num_values } - /// Compression for this column. + /// [`Compression`] for this column. + /// + /// This is a default value suitable for passing to [`WriterPropertiesBuilder::set_compression`]. + /// It is constructed from the `codec` field of the Parquet `ColumnMetaData` + /// + /// [`WriterPropertiesBuilder::set_compression`]: crate::file::properties::WriterPropertiesBuilder pub fn compression(&self) -> Compression { + self.compression.into() + } + + /// Returns the compression codec used when writing this column. + pub fn compression_codec(&self) -> CompressionCodec { self.compression } @@ -1234,7 +1241,7 @@ impl ColumnChunkMetaDataBuilder { file_path: None, file_offset: 0, num_values: 0, - compression: Compression::UNCOMPRESSED, + compression: CompressionCodec::UNCOMPRESSED, total_compressed_size: 0, total_uncompressed_size: 0, data_page_offset: 0, @@ -1285,8 +1292,14 @@ impl ColumnChunkMetaDataBuilder { self } - /// Sets compression. + /// Sets compression codec given a [`Compression`] configuration value. pub fn set_compression(mut self, value: Compression) -> Self { + self.0.compression = value.into(); + self + } + + /// Sets compression codec. + pub fn set_compression_codec(mut self, value: CompressionCodec) -> Self { self.0.compression = value; self } @@ -1820,7 +1833,7 @@ mod tests { )) .set_file_path("file_path".to_owned()) .set_num_values(1000) - .set_compression(Compression::SNAPPY) + .set_compression_codec(CompressionCodec::SNAPPY) .set_total_compressed_size(2000) .set_total_uncompressed_size(3000) .set_data_page_offset(4000) @@ -1860,7 +1873,7 @@ mod tests { )) .set_file_path("file_path".to_owned()) .set_num_values(1000) - .set_compression(Compression::SNAPPY) + .set_compression_codec(CompressionCodec::SNAPPY) .set_total_compressed_size(2000) .set_total_uncompressed_size(3000) .set_data_page_offset(4000) @@ -1903,7 +1916,7 @@ mod tests { [Encoding::PLAIN, Encoding::RLE].iter(), )) .set_num_values(1000) - .set_compression(Compression::SNAPPY) + .set_compression_codec(CompressionCodec::SNAPPY) .set_total_compressed_size(2000) .set_total_uncompressed_size(3000) .set_data_page_offset(4000) @@ -2034,9 +2047,9 @@ mod tests { .build(); #[cfg(not(feature = "encryption"))] - let base_expected_size = 2766; + let base_expected_size = 2734; #[cfg(feature = "encryption")] - let base_expected_size = 2934; + let base_expected_size = 2902; assert_eq!(parquet_meta.memory_size(), base_expected_size); @@ -2065,9 +2078,9 @@ mod tests { .build(); #[cfg(not(feature = "encryption"))] - let bigger_expected_size = 3192; + let bigger_expected_size = 3160; #[cfg(feature = "encryption")] - let bigger_expected_size = 3360; + let bigger_expected_size = 3328; // more set fields means more memory usage assert!(bigger_expected_size > base_expected_size); @@ -2114,7 +2127,7 @@ mod tests { .set_row_groups(row_group_meta.clone()) .build(); - let base_expected_size = 2058; + let base_expected_size = 2042; assert_eq!(parquet_meta_data.memory_size(), base_expected_size); let footer_key = "0123456789012345".as_bytes(); @@ -2140,7 +2153,7 @@ mod tests { .set_file_decryptor(Some(decryptor)) .build(); - let expected_size_with_decryptor = 3072; + let expected_size_with_decryptor = 3056; assert!(expected_size_with_decryptor > base_expected_size); assert_eq!( diff --git a/parquet/src/file/metadata/thrift/mod.rs b/parquet/src/file/metadata/thrift/mod.rs index 8c1147ff2c88..b68af0c48595 100644 --- a/parquet/src/file/metadata/thrift/mod.rs +++ b/parquet/src/file/metadata/thrift/mod.rs @@ -35,8 +35,8 @@ use crate::file::{ }; use crate::{ basic::{ - ColumnOrder, Compression, ConvertedType, Encoding, EncodingMask, LogicalType, PageType, - Repetition, Type, + ColumnOrder, CompressionCodec, ConvertedType, Encoding, EncodingMask, LogicalType, + PageType, Repetition, Type, }, data_type::{ByteArray, FixedLenByteArray, Int96}, errors::{ParquetError, Result}, @@ -460,7 +460,7 @@ fn read_column_metadata<'a>( } // 3: path_in_schema is redundant 4 => { - column.compression = Compression::read_thrift(&mut *prot)?; + column.compression = CompressionCodec::read_thrift(&mut *prot)?; seen_mask |= COL_META_CODEC; } 5 => { diff --git a/parquet/src/file/writer.rs b/parquet/src/file/writer.rs index beb83b71cebc..ec4d02a47d79 100644 --- a/parquet/src/file/writer.rs +++ b/parquet/src/file/writer.rs @@ -718,7 +718,7 @@ impl<'a, W: Write + Send> SerializedRowGroupWriter<'a, W> { let map_offset = |x| x - src_offset + write_offset as i64; let mut builder = ColumnChunkMetaData::builder(metadata.column_descr_ptr()) - .set_compression(metadata.compression()) + .set_compression_codec(metadata.compression_codec()) .set_encodings_mask(*metadata.encodings_mask()) .set_total_compressed_size(metadata.compressed_size()) .set_total_uncompressed_size(metadata.uncompressed_size()) @@ -1597,7 +1597,7 @@ mod tests { let desc = ColumnDescriptor::new(Arc::new(t), 0, 0, ColumnPath::new(vec![])); let meta = ColumnChunkMetaData::builder(Arc::new(desc)) - .set_compression(codec) + .set_compression_codec(codec.into()) .set_total_compressed_size(reader.len() as i64) .set_num_values(total_num_values) .build() diff --git a/parquet/src/schema/printer.rs b/parquet/src/schema/printer.rs index 4ad3b6b93e00..68398005b6a5 100644 --- a/parquet/src/schema/printer.rs +++ b/parquet/src/schema/printer.rs @@ -177,11 +177,7 @@ fn print_column_chunk_metadata(out: &mut dyn io::Write, cc_metadata: &ColumnChun writeln!(out, "file path: {file_path_str}"); writeln!(out, "file offset: {}", cc_metadata.file_offset()); writeln!(out, "num of values: {}", cc_metadata.num_values()); - writeln!( - out, - "compression: {}", - cc_metadata.compression().codec_to_string() - ); + writeln!(out, "compression: {}", cc_metadata.compression_codec()); writeln!( out, "total compressed size (in bytes): {}", From aa3c9d3ecc35891793d962015fd2cdff708ac63f Mon Sep 17 00:00:00 2001 From: Yu-Chuan Hung <86523891+CuteChuanChuan@users.noreply.github.com> Date: Fri, 8 May 2026 02:03:15 +0800 Subject: [PATCH 25/33] feat(parquet): add BloomFilterPropertiesBuilder (#9877) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Which issue does this PR close? - Closes #9667. # Rationale for this change No builder exists for `BloomFilterProperties`, so callers write `BloomFilterProperties { fpp, ndv }` literals — pinning field layout to the API and skipping fpp validation. `WriterPropertiesBuilder` also has no setter that takes a built `BloomFilterProperties`. # What changes are included in this PR? - `BloomFilterPropertiesBuilder` (`with_fpp`, `with_max_ndv`, `build`, `try_build`). Two entry points per discussion in the issue: `BloomFilterProperties::builder()` and `BloomFilterPropertiesBuilder::new()`. - `WriterPropertiesBuilder::set_bloom_filter_properties` + per-column variant. NDV from the passed-in struct is honoured (no row-group-size override). For dynamic NDV, keep using `set_bloom_filter_enabled` / `set_bloom_filter_fpp`. - Renamed `set_bloom_filter_ndv` → `set_bloom_filter_max_ndv` (also per-column). Old names are `#[deprecated(since = "59.0.0")]` aliases. # Are these changes tested? Yes — 10 new unit tests + a doc-test. # Are there any user-facing changes? Additive only. `set_bloom_filter_ndv` (and per-column) emit a deprecation warning pointing to `_max_ndv`. --- parquet/src/arrow/arrow_writer/mod.rs | 2 +- parquet/src/bin/parquet-rewrite.rs | 3 +- parquet/src/file/properties.rs | 353 ++++++++++++++++++++++++-- 3 files changed, 341 insertions(+), 17 deletions(-) diff --git a/parquet/src/arrow/arrow_writer/mod.rs b/parquet/src/arrow/arrow_writer/mod.rs index d1f1a3cffe14..04a798391429 100644 --- a/parquet/src/arrow/arrow_writer/mod.rs +++ b/parquet/src/arrow/arrow_writer/mod.rs @@ -2798,7 +2798,7 @@ mod tests { .set_bloom_filter_enabled(bloom_filter) .set_bloom_filter_position(bloom_filter_position); if let Some(ndv) = bloom_filter_ndv { - builder = builder.set_bloom_filter_ndv(ndv); + builder = builder.set_bloom_filter_max_ndv(ndv); } let props = builder.build(); diff --git a/parquet/src/bin/parquet-rewrite.rs b/parquet/src/bin/parquet-rewrite.rs index 7637f26d77ed..2428c491413e 100644 --- a/parquet/src/bin/parquet-rewrite.rs +++ b/parquet/src/bin/parquet-rewrite.rs @@ -384,7 +384,8 @@ fn main() { writer_properties_builder = writer_properties_builder.set_bloom_filter_fpp(value); } if let Some(value) = args.bloom_filter_ndv { - writer_properties_builder = writer_properties_builder.set_bloom_filter_ndv(value); + writer_properties_builder = + writer_properties_builder.set_bloom_filter_max_ndv(value); } if let Some(value) = args.bloom_filter_position { writer_properties_builder = diff --git a/parquet/src/file/properties.rs b/parquet/src/file/properties.rs index 1ab8b5c710ba..816973f83ca9 100644 --- a/parquet/src/file/properties.rs +++ b/parquet/src/file/properties.rs @@ -20,6 +20,7 @@ use crate::basic::{Compression, Encoding}; use crate::compression::{CodecOptions, CodecOptionsBuilder}; #[cfg(feature = "encryption")] use crate::encryption::encrypt::FileEncryptionProperties; +use crate::errors::{ParquetError, Result}; use crate::file::metadata::{KeyValue, SortingColumn}; use crate::schema::types::ColumnPath; use std::str::FromStr; @@ -1119,10 +1120,10 @@ impl WriterPropertiesBuilder { /// * If the bloom filter is enabled previously then it is a no-op. /// /// * If the bloom filter is not enabled, default values for ndv and fpp - /// value are used used. See [`set_bloom_filter_ndv`] and + /// value are used used. See [`set_bloom_filter_max_ndv`] and /// [`set_bloom_filter_fpp`] to further adjust the ndv and fpp. /// - /// [`set_bloom_filter_ndv`]: Self::set_bloom_filter_ndv + /// [`set_bloom_filter_max_ndv`]: Self::set_bloom_filter_max_ndv /// [`set_bloom_filter_fpp`]: Self::set_bloom_filter_fpp pub fn set_bloom_filter_enabled(mut self, value: bool) -> Self { self.default_column_properties @@ -1154,11 +1155,17 @@ impl WriterPropertiesBuilder { /// been called. /// /// [`set_bloom_filter_enabled`]: Self::set_bloom_filter_enabled - pub fn set_bloom_filter_ndv(mut self, value: u64) -> Self { + pub fn set_bloom_filter_max_ndv(mut self, value: u64) -> Self { self.default_column_properties.set_bloom_filter_ndv(value); self } + /// Deprecated alias for [`Self::set_bloom_filter_max_ndv`]. + #[deprecated(since = "59.0.0", note = "Use `set_bloom_filter_max_ndv` instead")] + pub fn set_bloom_filter_ndv(self, value: u64) -> Self { + self.set_bloom_filter_max_ndv(value) + } + // ---------------------------------------------------------------------- // Setters for a specific column @@ -1257,10 +1264,11 @@ impl WriterPropertiesBuilder { self } - /// Sets the number of distinct values for bloom filter for a specific column. + /// Sets the maximum expected number of distinct values for bloom filter for + /// a specific column. /// - /// Takes precedence over [`Self::set_bloom_filter_ndv`]. - pub fn set_column_bloom_filter_ndv(mut self, col: ColumnPath, value: u64) -> Self { + /// Takes precedence over [`Self::set_bloom_filter_max_ndv`]. + pub fn set_column_bloom_filter_max_ndv(mut self, col: ColumnPath, value: u64) -> Self { self.get_mut_props(col).set_bloom_filter_ndv(value); self } @@ -1280,6 +1288,41 @@ impl WriterPropertiesBuilder { .set_data_page_v2_compression_ratio_threshold(value); self } + + /// Deprecated alias for [`Self::set_column_bloom_filter_max_ndv`]. + #[deprecated( + since = "59.0.0", + note = "Use `set_column_bloom_filter_max_ndv` instead" + )] + pub fn set_column_bloom_filter_ndv(self, col: ColumnPath, value: u64) -> Self { + self.set_column_bloom_filter_max_ndv(col, value) + } + + /// Sets the [`BloomFilterProperties`] for all columns, implicitly enabling + /// the bloom filter. + /// + /// Both `fpp` and `ndv` from `value` are treated as explicit and will not + /// be overridden by the build-time row-group-size NDV fallback. For + /// dynamic NDV sizing (resolved to `max_row_group_row_count` at build + /// time), use [`Self::set_bloom_filter_enabled`] or + /// [`Self::set_bloom_filter_fpp`] instead. + pub fn set_bloom_filter_properties(mut self, value: BloomFilterProperties) -> Self { + self.default_column_properties + .set_bloom_filter_properties(value); + self + } + + /// Sets the [`BloomFilterProperties`] for a specific column. + /// + /// Takes precedence over [`Self::set_bloom_filter_properties`]. + pub fn set_column_bloom_filter_properties( + mut self, + col: ColumnPath, + value: BloomFilterProperties, + ) -> Self { + self.get_mut_props(col).set_bloom_filter_properties(value); + self + } } impl From for WriterPropertiesBuilder { @@ -1368,6 +1411,29 @@ impl Default for EnabledStatistics { /// maintaining the target `fpp`. See [`Sbbf::fold_to_target_fpp`] for details on the /// folding algorithm. /// +/// # Example +/// +/// ```rust +/// # use parquet::{ +/// # file::properties::{BloomFilterProperties, WriterProperties}, +/// # schema::types::ColumnPath, +/// # }; +/// // Build a BloomFilterProperties via the builder, then apply it to one column. +/// let bf = BloomFilterProperties::builder() +/// .with_fpp(0.01) +/// .with_max_ndv(10_000) +/// .build(); +/// +/// let props = WriterProperties::builder() +/// .set_column_bloom_filter_properties(ColumnPath::from("user_id"), bf.clone()) +/// .build(); +/// +/// assert_eq!( +/// props.bloom_filter_properties(&ColumnPath::from("user_id")), +/// Some(&bf) +/// ); +/// ``` +/// /// [`Sbbf::fold_to_target_fpp`]: crate::bloom_filter::Sbbf::fold_to_target_fpp #[derive(Debug, Clone, PartialEq)] pub struct BloomFilterProperties { @@ -1384,7 +1450,7 @@ pub struct BloomFilterProperties { pub fpp: f64, /// Maximum expected number of distinct values. Defaults to [`DEFAULT_BLOOM_FILTER_NDV`]. /// - /// You should set this value by calling [`WriterPropertiesBuilder::set_bloom_filter_ndv`]. + /// You should set this value by calling [`WriterPropertiesBuilder::set_bloom_filter_max_ndv`]. /// /// When not explicitly set via the builder, this defaults to /// [`max_row_group_row_count`](WriterProperties::max_row_group_row_count) (resolved at @@ -1401,7 +1467,7 @@ pub struct BloomFilterProperties { /// of the number of distinct values in a row group, it is recommended to set this value explicitly /// rather than relying on the default dynamic sizing based on `max_row_group_row_count`. /// If you do set this value explicitly it is probably best to set it for each column - /// individually via [`WriterPropertiesBuilder::set_column_bloom_filter_ndv`] rather than globally, + /// individually via [`WriterPropertiesBuilder::set_column_bloom_filter_max_ndv`] rather than globally, /// since different columns may have different numbers of distinct values. pub ndv: u64, } @@ -1415,6 +1481,93 @@ impl Default for BloomFilterProperties { } } +impl BloomFilterProperties { + /// Returns a new [`BloomFilterPropertiesBuilder`] for constructing + /// [`BloomFilterProperties`] with custom values. + pub fn builder() -> BloomFilterPropertiesBuilder { + BloomFilterPropertiesBuilder::new() + } +} + +/// Builder for [`BloomFilterProperties`]. +/// +/// Use [`BloomFilterProperties::builder`] or [`BloomFilterPropertiesBuilder::new`] +/// as the entry point. +#[derive(Debug, Clone, Default)] +pub struct BloomFilterPropertiesBuilder { + fpp: Option, + ndv: Option, +} + +impl BloomFilterPropertiesBuilder { + /// Returns a new builder with no fields set. + /// + /// Equivalent to [`BloomFilterProperties::builder`]. + pub fn new() -> Self { + Self::default() + } + + /// Sets the target false positive probability. + /// + /// The value must be in `(0.0, 1.0)` exclusively; this is validated at + /// build time by [`Self::build`] / [`Self::try_build`]. When unset, the + /// default is `0.05` (5%, see [`DEFAULT_BLOOM_FILTER_FPP`]). + pub fn with_fpp(mut self, fpp: f64) -> Self { + self.fpp = Some(fpp); + self + } + + /// Sets the maximum expected number of distinct values used to size the + /// bloom filter before folding. + /// + /// When unset, the default is `1_048_576` (see [`DEFAULT_BLOOM_FILTER_NDV`]), + /// which at the default fpp of 5% reserves roughly 1 MiB per column for the + /// filter bitset, derived as follows: + /// + /// ```text + /// ndv = 1,048,576, fpp = 0.05 + /// 0.05^(1/8) ≈ 0.6877 + /// 1 - 0.6877 ≈ 0.3123 + /// ln(0.3123) ≈ -1.164 + /// num_bits = -8 * 1,048,576 / -1.164 ≈ 7,206,000 bits + /// ≈ 900,750 bytes (~900 KB) + /// next_power_of_two(900 KB) = 1 MiB (= 1,048,576 bytes) + /// ``` + pub fn with_max_ndv(mut self, ndv: u64) -> Self { + self.ndv = Some(ndv); + self + } + + /// Builds [`BloomFilterProperties`]. + /// + /// Panics if the configured `fpp` is not in `(0.0, 1.0)` exclusive. + /// Use [`Self::try_build`] for a non-panicking alternative. + pub fn build(self) -> BloomFilterProperties { + self.try_build().unwrap_or_else(|e| panic!("{e}")) + } + + /// Builds [`BloomFilterProperties`], returning an error instead of + /// panicking when the configured `fpp` is not in `(0.0, 1.0)` exclusive. + pub fn try_build(self) -> Result { + let fpp = self.fpp.unwrap_or(DEFAULT_BLOOM_FILTER_FPP); + validate_bloom_filter_fpp(fpp).map_err(ParquetError::General)?; + let ndv = self.ndv.unwrap_or(DEFAULT_BLOOM_FILTER_NDV); + Ok(BloomFilterProperties { fpp, ndv }) + } +} + +/// Single source of truth for the bloom filter fpp range check, shared by +/// [`ColumnProperties::set_bloom_filter_fpp`] (panic path) and +/// [`BloomFilterPropertiesBuilder::try_build`] (Result path). +fn validate_bloom_filter_fpp(fpp: f64) -> std::result::Result<(), String> { + if !(fpp > 0.0 && fpp < 1.0) { + return Err(format!( + "fpp must be between 0.0 and 1.0 exclusive, got {fpp}" + )); + } + Ok(()) +} + /// Container for column properties that can be changed as part of writer. /// /// If a field is `None`, it means that no specific value has been set for this column, @@ -1500,11 +1653,9 @@ impl ColumnProperties { /// /// Panics if the `value` is not between 0 and 1 exclusive fn set_bloom_filter_fpp(&mut self, value: f64) { - assert!( - value > 0. && value < 1.0, - "fpp must be between 0 and 1 exclusive, got {value}" - ); - + if let Err(msg) = validate_bloom_filter_fpp(value) { + panic!("{msg}"); + } self.bloom_filter_properties .get_or_insert_with(Default::default) .fpp = value; @@ -1519,6 +1670,17 @@ impl ColumnProperties { self.bloom_filter_ndv_is_set = true; } + /// Sets the bloom filter properties for this column from a fully-built + /// [`BloomFilterProperties`], implicitly enabling the bloom filter. + /// + /// Both `fpp` and `ndv` from `value` are treated as explicit, so the + /// build-time row-group-size NDV fallback in + /// [`WriterPropertiesBuilder::build`] will not override them. + fn set_bloom_filter_properties(&mut self, value: BloomFilterProperties) { + self.bloom_filter_properties = Some(value); + self.bloom_filter_ndv_is_set = true; + } + /// Sets the Data Page v2 compression ratio threshold for this column. /// /// # Panics @@ -1839,7 +2001,7 @@ mod tests { .set_column_dictionary_enabled(ColumnPath::from("col"), true) .set_column_statistics_enabled(ColumnPath::from("col"), EnabledStatistics::Chunk) .set_column_bloom_filter_enabled(ColumnPath::from("col"), true) - .set_column_bloom_filter_ndv(ColumnPath::from("col"), 100_u64) + .set_column_bloom_filter_max_ndv(ColumnPath::from("col"), 100_u64) .set_column_bloom_filter_fpp(ColumnPath::from("col"), 0.1) .build(); @@ -1961,7 +2123,7 @@ mod tests { ); assert_eq!( WriterProperties::builder() - .set_bloom_filter_ndv(100) + .set_bloom_filter_max_ndv(100) .build() .bloom_filter_properties(&ColumnPath::from("col")), Some(&BloomFilterProperties { @@ -2009,6 +2171,30 @@ mod tests { .build(); } + #[test] + #[allow(deprecated)] + fn test_writer_properties_deprecated_bloom_filter_ndv_setters_still_work() { + let col = ColumnPath::from("col"); + let props = WriterProperties::builder() + .set_bloom_filter_ndv(100) + .set_column_bloom_filter_ndv(col.clone(), 200) + .build(); + assert_eq!( + props.bloom_filter_properties(&ColumnPath::from("other")), + Some(&BloomFilterProperties { + fpp: DEFAULT_BLOOM_FILTER_FPP, + ndv: 100, + }) + ); + assert_eq!( + props.bloom_filter_properties(&col), + Some(&BloomFilterProperties { + fpp: DEFAULT_BLOOM_FILTER_FPP, + ndv: 200, + }) + ); + } + #[test] fn test_writer_properties_column_dictionary_page_size_limit() { let props = WriterProperties::builder() @@ -2125,4 +2311,141 @@ mod tests { assert_eq!(custom, custom); assert_ne!(opts, custom); } + + #[test] + fn test_bloom_filter_builder_default() { + let props = BloomFilterProperties::builder().build(); + assert_eq!(props.fpp, DEFAULT_BLOOM_FILTER_FPP); + assert_eq!(props.ndv, DEFAULT_BLOOM_FILTER_NDV); + assert_eq!(props, BloomFilterProperties::default()); + assert_eq!( + BloomFilterPropertiesBuilder::new().build(), + BloomFilterProperties::default() + ); + } + + #[test] + fn test_bloom_filter_builder_explicit_fpp() { + let props = BloomFilterProperties::builder().with_fpp(0.01).build(); + assert_eq!(props.fpp, 0.01); + assert_eq!(props.ndv, DEFAULT_BLOOM_FILTER_NDV); + } + + #[test] + fn test_bloom_filter_builder_explicit_ndv() { + let props = BloomFilterProperties::builder().with_max_ndv(1000).build(); + assert_eq!(props.fpp, DEFAULT_BLOOM_FILTER_FPP); + assert_eq!(props.ndv, 1000); + } + + #[test] + fn test_bloom_filter_builder_validates_fpp() { + for wrong_val in [0.0_f64, 1.0, -0.5, 2.0] { + let result = std::panic::catch_unwind(|| { + BloomFilterProperties::builder().with_fpp(wrong_val).build() + }); + assert!( + result.is_err(), + "with_fpp({wrong_val}).build() should reject value outside (0, 1)" + ); + } + } + + #[test] + fn test_bloom_filter_builder_try_build_validates_fpp() { + for wrong_val in [0.0_f64, 1.0, -0.5, 2.0] { + let result = BloomFilterProperties::builder() + .with_fpp(wrong_val) + .try_build(); + assert!( + result.is_err(), + "try_build() should return Err for fpp outside (0, 1)" + ); + } + + let ok = BloomFilterProperties::builder() + .with_fpp(0.01) + .with_max_ndv(1000) + .try_build() + .expect("valid fpp should yield Ok"); + assert_eq!(ok.fpp, 0.01); + assert_eq!(ok.ndv, 1000); + } + + #[test] + fn test_column_specific_implicit_ndv_uses_row_group_size() { + let custom_row_group_size: usize = 7777; + let col = ColumnPath::from("col"); + let props = WriterProperties::builder() + .set_max_row_group_row_count(Some(custom_row_group_size)) + .set_column_bloom_filter_enabled(col.clone(), true) + .build(); + let bf = props + .bloom_filter_properties(&col) + .expect("bloom filter should be enabled for col"); + + assert_eq!(bf.ndv, custom_row_group_size as u64); + assert_eq!(bf.fpp, DEFAULT_BLOOM_FILTER_FPP); + } + + #[test] + fn test_set_bloom_filter_properties_applied_globally() { + let bf = BloomFilterProperties::builder() + .with_fpp(0.01) + .with_max_ndv(500) + .build(); + let props = WriterProperties::builder() + .set_bloom_filter_properties(bf.clone()) + .build(); + + assert_eq!( + props.bloom_filter_properties(&ColumnPath::from("a")), + Some(&bf), + ); + assert_eq!( + props.bloom_filter_properties(&ColumnPath::from("b")), + Some(&bf), + ); + } + + #[test] + fn test_set_column_bloom_filter_properties_overrides_global() { + let global = BloomFilterProperties::builder() + .with_fpp(0.01) + .with_max_ndv(500) + .build(); + let tailored = BloomFilterProperties::builder() + .with_fpp(0.02) + .with_max_ndv(1000) + .build(); + + let col = ColumnPath::from("col"); + let props = WriterProperties::builder() + .set_bloom_filter_properties(global.clone()) + .set_column_bloom_filter_properties(col.clone(), tailored.clone()) + .build(); + + assert_eq!(props.bloom_filter_properties(&col), Some(&tailored)); + assert_eq!( + props.bloom_filter_properties(&ColumnPath::from("other")), + Some(&global) + ); + } + + #[test] + fn test_set_bloom_filter_properties_preserve_explicit_ndv() { + let bf = BloomFilterProperties::builder().with_max_ndv(42).build(); + let props = WriterProperties::builder() + .set_max_row_group_row_count(Some(99_999)) + .set_bloom_filter_properties(bf) + .build(); + let result = props + .bloom_filter_properties(&ColumnPath::from("col")) + .expect("bloom filter should be enabled"); + + assert_eq!( + result.ndv, 42, + "explicit ndv must not be overridden by row-group-size fallback" + ); + } } From c1507ad20a3dad44353bd9fb4c489785298d10d8 Mon Sep 17 00:00:00 2001 From: Rostislav Rumenov Date: Thu, 7 May 2026 20:10:50 +0200 Subject: [PATCH 26/33] generic channel support for FlightClient (#9933) Allow FlightServiceClient to be parameterized over the underlying channel type, so users can wrap a tonic channel with custom interceptors or services. Motivation: Annotating outbound Flight requests with metadata (e.g. injecting OpenTelemetry trace context into headers) currently requires forking or wrapping at a higher level. Making the channel generic lets callers compose tower layers/interceptors idiomatically and propagate distributed tracing context without bespoke plumbing. --------- Co-authored-by: Rostislav Rumenov --- arrow-flight/src/client.rs | 291 +++++++++++++++++++++++++++++++++++-- 1 file changed, 280 insertions(+), 11 deletions(-) diff --git a/arrow-flight/src/client.rs b/arrow-flight/src/client.rs index dac086271cb7..b2059a81d0df 100644 --- a/arrow-flight/src/client.rs +++ b/arrow-flight/src/client.rs @@ -31,6 +31,7 @@ use futures::{ stream::{self, BoxStream}, }; use prost::Message; +use tonic::codegen::{Body, StdError}; use tonic::{metadata::MetadataMap, transport::Channel}; use crate::error::{FlightError, Result}; @@ -67,22 +68,28 @@ use crate::streams::{FallibleRequestStream, FallibleTonicResponseStream}; /// # } /// ``` #[derive(Debug)] -pub struct FlightClient { +pub struct FlightClient { /// Optional grpc header metadata to include with each request metadata: MetadataMap, /// The inner client - inner: FlightServiceClient, + inner: FlightServiceClient, } -impl FlightClient { - /// Creates a client client with the provided [`Channel`] - pub fn new(channel: Channel) -> Self { - Self::new_from_inner(FlightServiceClient::new(channel)) +impl FlightClient +where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, +{ + /// Creates a client with the provided transport + pub fn new(inner: T) -> Self { + Self::new_from_inner(FlightServiceClient::new(inner)) } /// Creates a new higher level client with the provided lower level client - pub fn new_from_inner(inner: FlightServiceClient) -> Self { + pub fn new_from_inner(inner: FlightServiceClient) -> Self { Self { metadata: MetadataMap::new(), inner, @@ -120,19 +127,19 @@ impl FlightClient { /// Return a reference to the underlying tonic /// [`FlightServiceClient`] - pub fn inner(&self) -> &FlightServiceClient { + pub fn inner(&self) -> &FlightServiceClient { &self.inner } /// Return a mutable reference to the underlying tonic /// [`FlightServiceClient`] - pub fn inner_mut(&mut self) -> &mut FlightServiceClient { + pub fn inner_mut(&mut self) -> &mut FlightServiceClient { &mut self.inner } /// Consume this client and return the underlying tonic /// [`FlightServiceClient`] - pub fn into_inner(self) -> FlightServiceClient { + pub fn into_inner(self) -> FlightServiceClient { self.inner } @@ -664,10 +671,272 @@ impl FlightClient { } /// return a Request, adding any configured metadata - fn make_request(&self, t: T) -> tonic::Request { + fn make_request(&self, t: R) -> tonic::Request { // Pass along metadata let mut request = tonic::Request::new(t); *request.metadata_mut() = self.metadata.clone(); request } } + +#[cfg(test)] +mod tests { + use super::FlightClient; + use crate::encode::FlightDataEncoderBuilder; + use crate::flight_service_server::{FlightService, FlightServiceServer}; + use crate::{ + Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightInfo, + HandshakeRequest, HandshakeResponse, PollInfo, PutResult, SchemaResult, Ticket, + }; + use arrow_array::{RecordBatch, UInt64Array}; + use bytes::Bytes; + use futures::{StreamExt, TryStreamExt, stream::BoxStream}; + use std::net::SocketAddr; + use std::sync::{Arc, Mutex}; + use std::time::Duration; + use tokio::net::TcpListener; + use tokio::task::JoinHandle; + use tonic::metadata::MetadataMap; + use tonic::service::interceptor::InterceptedService; + use tonic::transport::Channel; + use tonic::{Request, Response, Status, Streaming}; + use uuid::Uuid; + + /// Minimal `FlightService` that records request metadata and serves a + /// configured `do_get` response. Other RPCs return `Unimplemented`. + #[derive(Debug, Clone, Default)] + struct InterceptorTestServer { + state: Arc>, + } + + #[derive(Debug, Default)] + struct InterceptorTestState { + do_get_request: Option, + do_get_response: Option>>, + last_request_metadata: Option, + } + + impl InterceptorTestServer { + fn save_metadata(&self, request: &Request) { + self.state.lock().unwrap().last_request_metadata = Some(request.metadata().clone()); + } + + fn set_do_get_response(&self, response: Vec>) { + self.state.lock().unwrap().do_get_response = Some(response); + } + + fn take_do_get_request(&self) -> Option { + self.state.lock().unwrap().do_get_request.take() + } + + fn take_last_request_metadata(&self) -> Option { + self.state.lock().unwrap().last_request_metadata.take() + } + } + + #[tonic::async_trait] + impl FlightService for InterceptorTestServer { + type HandshakeStream = BoxStream<'static, Result>; + type ListFlightsStream = BoxStream<'static, Result>; + type DoGetStream = BoxStream<'static, Result>; + type DoPutStream = BoxStream<'static, Result>; + type DoActionStream = BoxStream<'static, Result>; + type ListActionsStream = BoxStream<'static, Result>; + type DoExchangeStream = BoxStream<'static, Result>; + + async fn do_get( + &self, + request: Request, + ) -> Result, Status> { + self.save_metadata(&request); + let mut state = self.state.lock().unwrap(); + state.do_get_request = Some(request.into_inner()); + + let batches = state + .do_get_response + .take() + .ok_or_else(|| Status::internal("no do_get response configured"))?; + let batch_stream = futures::stream::iter(batches).map_err(Into::into); + let stream = FlightDataEncoderBuilder::new() + .build(batch_stream) + .map_err(Into::into); + Ok(Response::new(stream.boxed())) + } + + async fn handshake( + &self, + _: Request>, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + async fn list_flights( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + async fn get_flight_info( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + async fn poll_flight_info( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + async fn get_schema( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + async fn do_put( + &self, + _: Request>, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + async fn do_action( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + async fn list_actions( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + async fn do_exchange( + &self, + _: Request>, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + } + + /// Spawns the test server on a background task and exposes a connected channel. + struct InterceptorTestFixture { + shutdown: Option>, + addr: SocketAddr, + handle: Option>>, + } + + impl InterceptorTestFixture { + async fn new(server: FlightServiceServer) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = tokio::sync::oneshot::channel(); + let shutdown_future = async move { + rx.await.ok(); + }; + let serve = tonic::transport::Server::builder() + .timeout(Duration::from_secs(30)) + .add_service(server) + .serve_with_incoming_shutdown( + tokio_stream::wrappers::TcpListenerStream::new(listener), + shutdown_future, + ); + let handle = tokio::task::spawn(serve); + Self { + shutdown: Some(tx), + addr, + handle: Some(handle), + } + } + + async fn channel(&self) -> Channel { + let url = format!("http://{}", self.addr); + tonic::transport::Endpoint::from_shared(url) + .expect("valid endpoint") + .timeout(Duration::from_secs(30)) + .connect() + .await + .expect("error connecting to server") + } + + async fn shutdown_and_wait(mut self) { + if let Some(tx) = self.shutdown.take() { + tx.send(()).expect("server quit early"); + } + if let Some(handle) = self.handle.take() { + handle + .await + .expect("task join error (panic?)") + .expect("server error at shutdown"); + } + } + } + + /// Integration test: a tonic [`Channel`] wrapped in an [`InterceptedService`] + /// that injects a custom header is passed to [`FlightClient`], and the server + /// observes the header on the request. + #[tokio::test] + async fn test_flight_client_with_intercepted_channel_passes_custom_header() { + let test_server = InterceptorTestServer::default(); + let fixture = + InterceptorTestFixture::new(FlightServiceServer::new(test_server.clone())).await; + + let channel = fixture.channel().await; + + let header_name = "x-random-header"; + let header_value = format!("random-{}", Uuid::new_v4()); + let header_value_for_interceptor = header_value.clone(); + + let interceptor = move |mut req: Request<()>| -> Result, Status> { + req.metadata_mut().insert( + header_name, + header_value_for_interceptor + .parse() + .expect("valid metadata value"), + ); + Ok(req) + }; + + let intercepted = InterceptedService::new(channel, interceptor); + let mut client = FlightClient::new(intercepted); + + let ticket = Ticket { + ticket: Bytes::from("dummy-ticket"), + }; + + let batch = RecordBatch::try_from_iter(vec![( + "col", + Arc::new(UInt64Array::from_iter([1, 2, 3, 4])) as _, + )]) + .unwrap(); + + test_server.set_do_get_response(vec![Ok(batch.clone())]); + + let response_stream = client + .do_get(ticket.clone()) + .await + .expect("error making do_get request"); + + let response: Vec = response_stream + .try_collect() + .await + .expect("error streaming data"); + + assert_eq!(response, vec![batch]); + assert_eq!(test_server.take_do_get_request(), Some(ticket)); + + let metadata = test_server + .take_last_request_metadata() + .expect("server received headers") + .into_headers(); + + let received = metadata + .get(header_name) + .expect("interceptor header missing on server") + .to_str() + .expect("ascii header value"); + assert_eq!(received, header_value); + + fixture.shutdown_and_wait().await; + } +} From 3c71d928b65da7a85e96ce9e81d6c0a67d4be193 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20Subiotto=20Marqu=C3=A9s?= Date: Thu, 7 May 2026 20:12:08 +0200 Subject: [PATCH 27/33] perf[arrow-select]: add specialized REE interleave (#9856) Benchmarks for this PR are in #9849. They have been separated out so we can compare this PR to main once the benchmarks have merged. The specialized interleave works by preserving run ends as much as possible by coalescing groups of adjacent logical indices pointing to the same source and calling interleave on the run end values. Future work could additionally coalesce values across sources, but this requires a value equality check. # Which issue does this PR close? - None # Rationale for this change interleave_fallback on REE arrays is slow # What changes are included in this PR? A specialized REE interleave implementation # Are these changes tested? Yes, by existing tests. # Are there any user-facing changes? Signed-off-by: Alfonso Subiotto Marques --- arrow-select/src/interleave.rs | 71 ++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/arrow-select/src/interleave.rs b/arrow-select/src/interleave.rs index f5904bc171ee..1bd34a7f3a58 100644 --- a/arrow-select/src/interleave.rs +++ b/arrow-select/src/interleave.rs @@ -108,6 +108,12 @@ pub fn interleave( DataType::Struct(fields) => interleave_struct(fields, values, indices), DataType::List(field) => interleave_list::(values, indices, field), DataType::LargeList(field) => interleave_list::(values, indices, field), + DataType::RunEndEncoded(r, _) => match r.data_type() { + DataType::Int16 => interleave_run_end::(values, indices), + DataType::Int32 => interleave_run_end::(values, indices), + DataType::Int64 => interleave_run_end::(values, indices), + t => unreachable!("illegal run-end type {t}"), + }, _ => interleave_fallback(values, indices) } } @@ -411,6 +417,71 @@ fn interleave_list( Ok(Arc::new(list_array)) } +/// Specialized [`interleave`] for [`RunArray`]. +fn interleave_run_end( + values: &[&dyn Array], + indices: &[(usize, usize)], +) -> Result { + if indices.is_empty() { + return Ok(new_empty_array(values[0].data_type())); + } + + let n = indices.len(); + R::Native::from_usize(n).ok_or_else(|| { + ArrowError::ComputeError(format!( + "interleave_run_end: output length {n} does not fit run-end type" + )) + })?; + + let runs: Vec<&RunArray> = values.iter().map(|a| a.as_run::()).collect(); + let value_arrays: Vec<&dyn Array> = runs.iter().map(|r| r.values().as_ref()).collect(); + + // Resolve each (array, logical_row) to (array, physical_row), so we can + // lookup physical indices by batch. + let mut phys_pairs: Vec<(usize, usize)> = vec![(0, 0); n]; + let mut grouped: Vec<(Vec, Vec)> = + (0..runs.len()).map(|_| (Vec::new(), Vec::new())).collect(); + for (out_pos, &(arr, row)) in indices.iter().enumerate() { + let row = R::Native::from_usize(row).ok_or_else(|| { + ArrowError::InvalidArgumentError(format!( + "interleave_run_end: row index {row} not representable as run-end type {}", + R::DATA_TYPE + )) + })?; + grouped[arr].0.push(row); + grouped[arr].1.push(out_pos); + } + for (arr_idx, (logical_rows, out_positions)) in grouped.into_iter().enumerate() { + let phys = runs[arr_idx].get_physical_indices(&logical_rows)?; + for (p, out_pos) in phys.iter().zip(out_positions.iter()) { + phys_pairs[*out_pos] = (arr_idx, *p); + } + } + + // Coalesce by physical-pair equality only: emit a new run when the + // (array_idx, physical_idx) pair changes between adjacent output rows. + // TODO: We could perform an equality check across sources to extend the + // output run, but we can't call make_comparator from this crate. + let mut run_ends_buf: Vec = Vec::with_capacity(n); + let mut dedup_pairs: Vec<(usize, usize)> = Vec::with_capacity(n); + dedup_pairs.push(phys_pairs[0]); + for i in 1..n { + if phys_pairs[i] != phys_pairs[i - 1] { + run_ends_buf.push(R::Native::from_usize(i).unwrap()); + dedup_pairs.push(phys_pairs[i]); + } + } + run_ends_buf.push(R::Native::from_usize(n).unwrap()); + + let taken_values = interleave(&value_arrays, &dedup_pairs)?; + let run_ends = PrimitiveArray::::from_iter_values(run_ends_buf); + + Ok(Arc::new(RunArray::::try_new( + &run_ends, + taken_values.as_ref(), + )?)) +} + /// Fallback implementation of interleave using [`MutableArrayData`] fn interleave_fallback( values: &[&dyn Array], From 6ce4bc899644c1b0816072f3260d835a0c14d148 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Thu, 7 May 2026 11:24:08 -0700 Subject: [PATCH 28/33] Validate encoded Thrift lists match the schema (#9924) # Which issue does this PR close? - Part of #9923. # Rationale for this change A first attempt at adding some validation. This will check that the encoded list element type matches what is expected from the Parquet schema. # What changes are included in this PR? Adds an `ELEMENT_TYPE` to the `ReadThrift` trait for use in validating data types in `read_thrift_vec`. # Are these changes tested? Should be covered by existing. These changes also cause an earlier error detection in an existing test of malformed data. # Are there any user-facing changes? No, just improves error handling --- parquet/src/basic.rs | 4 ++- parquet/src/file/metadata/thrift/mod.rs | 9 ++++++- parquet/src/file/page_index/offset_index.rs | 3 ++- parquet/src/file/serialized_reader.rs | 2 +- parquet/src/parquet_thrift.rs | 30 ++++++++++++++++++++- parquet/tests/arrow_reader/bad_data.rs | 2 +- 6 files changed, 44 insertions(+), 6 deletions(-) diff --git a/parquet/src/basic.rs b/parquet/src/basic.rs index cc7a16eca063..17e5b9c321bb 100644 --- a/parquet/src/basic.rs +++ b/parquet/src/basic.rs @@ -28,7 +28,7 @@ pub use crate::compression::{BrotliLevel, GzipLevel, ZstdLevel}; use crate::file::metadata::HeapSize; use crate::parquet_thrift::{ ElementType, FieldType, ReadThrift, ThriftCompactInputProtocol, ThriftCompactOutputProtocol, - WriteThrift, WriteThriftField, + WriteThrift, WriteThriftField, validate_list_type, }; use crate::{thrift_enum, thrift_struct, thrift_union_all_empty, write_thrift_field}; @@ -771,6 +771,8 @@ impl<'a, R: ThriftCompactInputProtocol<'a>> ReadThrift<'a, R> for EncodingMask { // This reads a Thrift `list` and turns it into a bitmask let list_ident = prot.read_list_begin()?; + // check for enum (encoded as I32) + validate_list_type(ElementType::I32, &list_ident)?; for _ in 0..list_ident.size { let val = Encoding::read_thrift(prot)?; mask |= 1 << val as i32; diff --git a/parquet/src/file/metadata/thrift/mod.rs b/parquet/src/file/metadata/thrift/mod.rs index b68af0c48595..75fcad871e2f 100644 --- a/parquet/src/file/metadata/thrift/mod.rs +++ b/parquet/src/file/metadata/thrift/mod.rs @@ -51,7 +51,7 @@ use crate::{ parquet_thrift::{ ElementType, FieldType, ReadThrift, ThriftCompactInputProtocol, ThriftCompactOutputProtocol, ThriftSliceInputProtocol, WriteThrift, WriteThriftField, - read_thrift_vec, + read_thrift_vec, validate_list_type, }, schema::types::{ ColumnDescriptor, SchemaDescriptor, TypePtr, num_nodes, parquet_schema_from_array, @@ -387,6 +387,9 @@ fn read_encoding_stats_as_mask<'a>( // read the vector of stats, setting mask bits for data pages let mut mask = 0i32; let list_ident = prot.read_list_begin()?; + // check for PageEncodingStats struct + validate_list_type(ElementType::Struct, &list_ident)?; + for _ in 0..list_ident.size { let pes = PageEncodingStats::read_thrift(prot)?; match pes.page_type { @@ -652,6 +655,8 @@ fn read_row_group( match field_ident.id { 1 => { let list_ident = prot.read_list_begin()?; + // check for list of struct + validate_list_type(ElementType::Struct, &list_ident)?; if schema_descr.num_columns() != list_ident.size as usize { return Err(general_err!( "Column count mismatch. Schema has {} columns while Row Group has {}", @@ -801,6 +806,8 @@ pub(crate) fn parquet_metadata_from_bytes( } let schema_descr = schema_descr.as_ref().unwrap(); let list_ident = prot.read_list_begin()?; + // check for list of struct + validate_list_type(ElementType::Struct, &list_ident)?; let mut rg_vec = Vec::with_capacity(list_ident.size as usize); // Read row groups and handle ordinal assignment diff --git a/parquet/src/file/page_index/offset_index.rs b/parquet/src/file/page_index/offset_index.rs index b1e30dd4590c..06d21efb68c8 100644 --- a/parquet/src/file/page_index/offset_index.rs +++ b/parquet/src/file/page_index/offset_index.rs @@ -23,7 +23,7 @@ use std::io::Write; use crate::parquet_thrift::{ ElementType, FieldType, ReadThrift, ThriftCompactInputProtocol, ThriftCompactOutputProtocol, - WriteThrift, WriteThriftField, read_thrift_vec, + WriteThrift, WriteThriftField, read_thrift_vec, validate_list_type, }; use crate::{ errors::{ParquetError, Result}, @@ -90,6 +90,7 @@ impl OffsetIndexMetaData { // we have to do this manually because we want to use the fast PageLocation decoder let list_ident = prot.read_list_begin()?; + validate_list_type(ElementType::Struct, &list_ident)?; let mut page_locations = Vec::with_capacity(list_ident.size as usize); for _ in 0..list_ident.size { page_locations.push(read_page_location(prot)?); diff --git a/parquet/src/file/serialized_reader.rs b/parquet/src/file/serialized_reader.rs index 254ccb779a4a..4b71e3c14e6d 100644 --- a/parquet/src/file/serialized_reader.rs +++ b/parquet/src/file/serialized_reader.rs @@ -2117,7 +2117,7 @@ mod tests { let ret = SerializedFileReader::new(Bytes::copy_from_slice(&data)); assert_eq!( ret.err().unwrap().to_string(), - "Parquet error: Received empty union from remote ColumnOrder" + "Parquet error: Expected list element type of Struct but got List" ); } diff --git a/parquet/src/parquet_thrift.rs b/parquet/src/parquet_thrift.rs index e621f4c4989b..b9d69b9d52e8 100644 --- a/parquet/src/parquet_thrift.rs +++ b/parquet/src/parquet_thrift.rs @@ -702,9 +702,10 @@ impl<'a, R: ThriftCompactInputProtocol<'a>> ReadThrift<'a, R> for &'a [u8] { pub(crate) fn read_thrift_vec<'a, T, R>(prot: &mut R) -> Result> where R: ThriftCompactInputProtocol<'a>, - T: ReadThrift<'a, R>, + T: ReadThrift<'a, R> + WriteThrift, { let list_ident = prot.read_list_begin()?; + validate_list_type(T::ELEMENT_TYPE, &list_ident)?; let mut res = Vec::with_capacity(list_ident.size as usize); for _ in 0..list_ident.size { let val = T::read_thrift(prot)?; @@ -713,6 +714,17 @@ where Ok(res) } +pub(crate) fn validate_list_type(expected: ElementType, got: &ListIdentifier) -> Result<()> { + if got.element_type != expected { + return Err(general_err!( + "Expected list element type of {:?} but got {:?}", + expected, + got.element_type + )); + } + Ok(()) +} + ///////////////////////// // thrift compact output @@ -1155,4 +1167,20 @@ pub(crate) mod tests { let result = prot.read_list_begin(); assert!(result.is_err(), "expected error, got {result:?}"); } + + #[test] + fn test_read_list_wrong_type() { + // list header: 4 elements of `Boolean` + let data = [0x42, 0x01]; + let mut prot = ThriftSliceInputProtocol::new(&data); + // try to read as list + let result = read_thrift_vec::(&mut prot); + println!("{result:?}"); + assert!( + result + .unwrap_err() + .to_string() + .contains("Expected list element type of I32 but got Bool") + ); + } } diff --git a/parquet/tests/arrow_reader/bad_data.rs b/parquet/tests/arrow_reader/bad_data.rs index d9b0d89e2c5f..56fddf505dac 100644 --- a/parquet/tests/arrow_reader/bad_data.rs +++ b/parquet/tests/arrow_reader/bad_data.rs @@ -98,7 +98,7 @@ fn test_arrow_gh_41317() { let err = read_file("ARROW-GH-41317.parquet").unwrap_err(); assert_eq!( err.to_string(), - "External: Parquet argument error: Parquet error: StructArrayReader out of sync in read_records, expected 5 read, got 2" + "Parquet error: Expected list element type of I32 but got I16" ); } From 7abb2255f2de045ca8dcd12e1e46377e11de5b9f Mon Sep 17 00:00:00 2001 From: Hippolyte Barraud Date: Thu, 7 May 2026 14:25:50 -0400 Subject: [PATCH 29/33] bench(parquet): add `ListArray` benchmarks for runtime and peak memory (#9846) # Which issue does this PR close? - Contributes to #9731 - Dependency of #9848 # Rationale for this change See #9848 Existing benchmarks have some gaps in the types of columns they exercise. Additionally, I would like to improve the memory efficiency of the read/decode path in terms of RSS requirements, especially for sparse inputs and we currently do not have any infrastructure to measure that. # What changes are included in this PR? Extend the existing `arrow_reader` runtime benchmarks with `Int32` and `FixedBinary32` list columns alongside the existing `StringList`, with parameterized null density (0%, 50%, 90%, 99%). The prior benchmarks only covered string lists, which didn't surface costs specific to fixed-width and primitive element types. Add a new `arrow_reader_peak_memory` benchmark that measures peak heap usage during `ListArrayReader::consume_batch` using a thread-local tracking allocator. It captures how RSS-efficient we are when materializing a column into its final Arrow in-memory representation. # Are these changes tested? All tests passing. # Are there any user-facing changes? None. Signed-off-by: Hippolyte Barraud Co-authored-by: Andrew Lamb --- parquet/Cargo.toml | 5 + parquet/benches/arrow_reader.rs | 221 +++++++- parquet/benches/arrow_reader_peak_memory.rs | 597 ++++++++++++++++++++ 3 files changed, 808 insertions(+), 15 deletions(-) create mode 100644 parquet/benches/arrow_reader_peak_memory.rs diff --git a/parquet/Cargo.toml b/parquet/Cargo.toml index 9f4d2a33df0c..2edeec40635e 100644 --- a/parquet/Cargo.toml +++ b/parquet/Cargo.toml @@ -289,5 +289,10 @@ required-features = ["arrow"] name = "bloom_filter" harness = false +[[bench]] +name = "arrow_reader_peak_memory" +required-features = ["arrow", "test_common", "experimental"] +harness = false + [lib] bench = false diff --git a/parquet/benches/arrow_reader.rs b/parquet/benches/arrow_reader.rs index 9325a1faf5b6..888ff5083ad0 100644 --- a/parquet/benches/arrow_reader.rs +++ b/parquet/benches/arrow_reader.rs @@ -95,6 +95,16 @@ fn build_test_schema() -> SchemaDescPtr { OPTIONAL GROUP optional_struct_optional_int32_leaf { OPTIONAL INT32 element; } + OPTIONAL GROUP int32_list (LIST) { + repeated group list { + optional INT32 element; + } + } + OPTIONAL GROUP fixed32_list (LIST) { + repeated group list { + optional FIXED_LEN_BYTE_ARRAY(32) element; + } + } } "; parse_message_type(message_type) @@ -668,6 +678,150 @@ fn build_string_list_page_iterator( InMemoryPageIterator::new(pages) } +fn build_int32_list_page_iterator( + column_desc: ColumnDescPtr, + null_density: f32, +) -> impl PageIterator + Clone { + let max_def_level = column_desc.max_def_level(); + let max_rep_level = column_desc.max_rep_level(); + assert_eq!(max_def_level, 3); + assert_eq!(max_rep_level, 1); + + let mut rng = seedable_rng(); + let mut pages: Vec> = Vec::new(); + for _i in 0..NUM_ROW_GROUPS { + let mut column_chunk_pages = Vec::new(); + for _j in 0..PAGES_PER_GROUP { + let mut values: Vec = Vec::with_capacity(VALUES_PER_PAGE * MAX_LIST_LEN); + let mut def_levels = Vec::with_capacity(VALUES_PER_PAGE * MAX_LIST_LEN); + let mut rep_levels = Vec::with_capacity(VALUES_PER_PAGE * MAX_LIST_LEN); + for _k in 0..VALUES_PER_PAGE { + rep_levels.push(0); + if rng.random::() < null_density { + def_levels.push(0); + continue; + } + let len = rng.random_range(0..MAX_LIST_LEN); + if len == 0 { + def_levels.push(1); + continue; + } + + (1..len).for_each(|_| rep_levels.push(1)); + + for _l in 0..len { + if rng.random::() < null_density { + def_levels.push(2); + } else { + def_levels.push(3); + values.push(rng.random()); + } + } + } + let mut page_builder = + DataPageBuilderImpl::new(column_desc.clone(), values.len() as u32, true); + page_builder.add_rep_levels(max_rep_level, &rep_levels); + page_builder.add_def_levels(max_def_level, &def_levels); + page_builder.add_values::(Encoding::PLAIN, &values); + column_chunk_pages.push(page_builder.consume()); + } + pages.push(column_chunk_pages); + } + + InMemoryPageIterator::new(pages) +} + +fn create_int32_list_reader( + page_iterator: impl PageIterator + 'static, + column_desc: ColumnDescPtr, +) -> Box { + use parquet::arrow::array_reader::PrimitiveArrayReader; + let items = Box::new( + PrimitiveArrayReader::::new( + Box::new(page_iterator), + column_desc, + None, + DEFAULT_BATCH_SIZE, + ) + .unwrap(), + ) as Box; + let field = Field::new_list_field(DataType::Int32, true); + let data_type = DataType::List(Arc::new(field)); + Box::new(ListArrayReader::::new(items, data_type, 2, 1, true)) +} + +const FIXED_BYTE_LEN: usize = 32; + +fn build_fixed32_list_page_iterator( + column_desc: ColumnDescPtr, + null_density: f32, +) -> impl PageIterator + Clone { + let max_def_level = column_desc.max_def_level(); + let max_rep_level = column_desc.max_rep_level(); + assert_eq!(max_def_level, 3); + assert_eq!(max_rep_level, 1); + + let mut rng = seedable_rng(); + let mut pages: Vec> = Vec::new(); + for _i in 0..NUM_ROW_GROUPS { + let mut column_chunk_pages = Vec::new(); + for _j in 0..PAGES_PER_GROUP { + let mut values: Vec = + Vec::with_capacity(VALUES_PER_PAGE * MAX_LIST_LEN); + let mut def_levels = Vec::with_capacity(VALUES_PER_PAGE * MAX_LIST_LEN); + let mut rep_levels = Vec::with_capacity(VALUES_PER_PAGE * MAX_LIST_LEN); + for _k in 0..VALUES_PER_PAGE { + rep_levels.push(0); + if rng.random::() < null_density { + def_levels.push(0); + continue; + } + let len = rng.random_range(0..MAX_LIST_LEN); + if len == 0 { + def_levels.push(1); + continue; + } + (1..len).for_each(|_| rep_levels.push(1)); + for _l in 0..len { + if rng.random::() < null_density { + def_levels.push(2); + } else { + def_levels.push(3); + let mut buf = vec![0u8; FIXED_BYTE_LEN]; + rng.fill(&mut buf[..]); + values.push(buf.into()); + } + } + } + let mut page_builder = + DataPageBuilderImpl::new(column_desc.clone(), values.len() as u32, true); + page_builder.add_rep_levels(max_rep_level, &rep_levels); + page_builder.add_def_levels(max_def_level, &def_levels); + page_builder.add_values::(Encoding::PLAIN, &values); + column_chunk_pages.push(page_builder.consume()); + } + pages.push(column_chunk_pages); + } + + InMemoryPageIterator::new(pages) +} + +fn create_fixed32_list_reader( + page_iterator: impl PageIterator + 'static, + column_desc: ColumnDescPtr, +) -> Box { + let items = make_fixed_len_byte_array_reader( + Box::new(page_iterator), + column_desc, + None, + DEFAULT_BATCH_SIZE, + ) + .unwrap(); + let field = Field::new_list_field(DataType::FixedSizeBinary(FIXED_BYTE_LEN as i32), true); + let data_type = DataType::List(Arc::new(field)); + Box::new(ListArrayReader::::new(items, data_type, 2, 1, true)) +} + fn bench_array_reader(mut array_reader: Box) -> usize { // test procedure: read data in batches of 8192 until no more data let mut total_count = 0; @@ -1645,6 +1799,8 @@ fn add_benches(c: &mut Criterion) { let optional_uint64_column_desc = schema.column(38); let mandatory_struct_optional_in32_column_desc = schema.column(39); let optional_struct_optional_in32_column_desc = schema.column(40); + let int32_list_desc = schema.column(41); + let fixed32_list_desc = schema.column(42); // primitive / int32 benchmarks // ============================= @@ -2228,24 +2384,59 @@ fn add_benches(c: &mut Criterion) { // list benchmarks //============================== - let list_data = build_string_list_page_iterator(string_list_desc.clone(), 0.); - let mut group = c.benchmark_group("arrow_array_reader/ListArray"); - group.bench_function("plain encoded optional strings no NULLs", |b| { - b.iter(|| { - let reader = create_string_list_reader(list_data.clone(), string_list_desc.clone()); - count = bench_array_reader(reader); + let mut group = c.benchmark_group("arrow_array_reader/ListArray/StringList"); + for (label, null_density) in [ + ("no NULLs", 0.0), + ("half NULLs", 0.5), + ("90pct NULLs", 0.9), + ("99pct NULLs", 0.99), + ] { + let list_data = build_string_list_page_iterator(string_list_desc.clone(), null_density); + group.bench_function(label, |b| { + b.iter(|| { + let reader = create_string_list_reader(list_data.clone(), string_list_desc.clone()); + count = bench_array_reader(reader); + }); + assert_eq!(count, EXPECTED_VALUE_COUNT); }); - assert_eq!(count, EXPECTED_VALUE_COUNT); - }); - let list_data = build_string_list_page_iterator(string_list_desc.clone(), 0.5); - group.bench_function("plain encoded optional strings half NULLs", |b| { - b.iter(|| { - let reader = create_string_list_reader(list_data.clone(), string_list_desc.clone()); - count = bench_array_reader(reader); + } + group.finish(); + + let mut group = c.benchmark_group("arrow_array_reader/ListArray/Int32List"); + for (label, null_density) in [ + ("no NULLs", 0.0), + ("half NULLs", 0.5), + ("90pct NULLs", 0.9), + ("99pct NULLs", 0.99), + ] { + let list_data = build_int32_list_page_iterator(int32_list_desc.clone(), null_density); + group.bench_function(label, |b| { + b.iter(|| { + let reader = create_int32_list_reader(list_data.clone(), int32_list_desc.clone()); + count = bench_array_reader(reader); + }); + assert_eq!(count, EXPECTED_VALUE_COUNT); }); - assert_eq!(count, EXPECTED_VALUE_COUNT); - }); + } + group.finish(); + let mut group = c.benchmark_group("arrow_array_reader/ListArray/Fixed32List"); + for (label, null_density) in [ + ("no NULLs", 0.0), + ("half NULLs", 0.5), + ("90pct NULLs", 0.9), + ("99pct NULLs", 0.99), + ] { + let list_data = build_fixed32_list_page_iterator(fixed32_list_desc.clone(), null_density); + group.bench_function(label, |b| { + b.iter(|| { + let reader = + create_fixed32_list_reader(list_data.clone(), fixed32_list_desc.clone()); + count = bench_array_reader(reader); + }); + assert_eq!(count, EXPECTED_VALUE_COUNT); + }); + } group.finish(); // fixed_len_byte_array benchmarks diff --git a/parquet/benches/arrow_reader_peak_memory.rs b/parquet/benches/arrow_reader_peak_memory.rs new file mode 100644 index 000000000000..6486bf2da0bf --- /dev/null +++ b/parquet/benches/arrow_reader_peak_memory.rs @@ -0,0 +1,597 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Criterion benchmark measuring memory usage of ListArrayReader for sparse +//! list columns. +//! +//! Uses a global allocator with thread-local memory tracking so that +//! concurrent criterion threads don't interfere with measurements. + +use std::alloc::Layout; +use std::cell::Cell; +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field}; +use criterion::measurement::{Measurement, ValueFormatter}; +use criterion::{BenchmarkGroup, Criterion, Throughput, criterion_group, criterion_main}; +use parquet::arrow::array_reader::make_byte_array_reader; +use parquet::arrow::array_reader::{ + ArrayReader, ListArrayReader, PrimitiveArrayReader, make_fixed_len_byte_array_reader, +}; +use parquet::arrow::arrow_reader::DEFAULT_BATCH_SIZE; +use parquet::basic::Encoding; +use parquet::column::page::PageIterator; +use parquet::data_type::{ByteArrayType, DoubleType, FixedLenByteArrayType, Int32Type}; +use parquet::schema::parser::parse_message_type; +use parquet::schema::types::{ColumnDescPtr, SchemaDescriptor}; +use parquet::util::{DataPageBuilder, DataPageBuilderImpl, InMemoryPageIterator}; +use rand::{Rng, SeedableRng, rngs::StdRng}; + +// --------------------------------------------------------------------------- +// Thread-local tracking allocator +// --------------------------------------------------------------------------- + +thread_local! { + static LIVE_BYTES: Cell = const { Cell::new(0) }; + static PEAK_BYTES: Cell = const { Cell::new(0) }; + static ALLOCATED_BYTES: Cell = const { Cell::new(0) }; +} + +struct TrackingAllocator { + inner: std::alloc::System, +} + +#[global_allocator] +static GLOBAL: TrackingAllocator = TrackingAllocator { + inner: std::alloc::System, +}; + +fn add_live_bytes(size: usize) { + LIVE_BYTES.with(|live| { + let new = live.get().saturating_add(size); + live.set(new); + PEAK_BYTES.with(|peak| { + if new > peak.get() { + peak.set(new); + } + }); + }); +} + +fn subtract_live_bytes(size: usize) { + LIVE_BYTES.with(|live| { + live.set(live.get().saturating_sub(size)); + }); +} + +fn add_allocated_bytes(size: usize) { + ALLOCATED_BYTES.with(|allocated| { + allocated.set(allocated.get().saturating_add(size)); + }); +} + +#[allow(unsafe_code)] +unsafe impl std::alloc::GlobalAlloc for TrackingAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let ptr = unsafe { self.inner.alloc(layout) }; + if !ptr.is_null() { + add_live_bytes(layout.size()); + add_allocated_bytes(layout.size()); + } + ptr + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + subtract_live_bytes(layout.size()); + unsafe { self.inner.dealloc(ptr, layout) }; + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + let new_ptr = unsafe { self.inner.realloc(ptr, layout, new_size) }; + if !new_ptr.is_null() { + let old_size = layout.size(); + add_allocated_bytes(new_size); + if new_size > old_size { + add_live_bytes(new_size - old_size); + } else { + subtract_live_bytes(old_size - new_size); + } + } + new_ptr + } +} + +fn reset_peak() { + PEAK_BYTES.with(|peak| { + LIVE_BYTES.with(|live| { + peak.set(live.get()); + }); + }); +} + +fn peak_bytes() -> usize { + PEAK_BYTES.with(|peak| peak.get()) +} + +fn live_bytes() -> usize { + LIVE_BYTES.with(|live| live.get()) +} + +fn reset_allocated() { + ALLOCATED_BYTES.with(|allocated| allocated.set(0)); +} + +fn allocated_bytes() -> usize { + ALLOCATED_BYTES.with(|allocated| allocated.get()) +} + +// --------------------------------------------------------------------------- +// Criterion custom measurements +// --------------------------------------------------------------------------- + +struct BytesFormatter; + +const BYTE_UNITS: &[(u32, &str)] = &[ + (60, "EiB"), + (50, "PiB"), + (40, "TiB"), + (30, "GiB"), + (20, "MiB"), + (10, "KiB"), + (0, "B"), +]; + +fn bytes_per_unit(exponent: u32) -> f64 { + (1_u64 << exponent) as f64 +} + +fn scale_bytes(typical: f64, values: &mut [f64]) -> &'static str { + for &(exponent, unit) in BYTE_UNITS { + let scale = bytes_per_unit(exponent); + if typical >= scale { + for v in values.iter_mut() { + *v /= scale; + } + return unit; + } + } + unreachable!("BYTE_UNITS contains B") +} + +impl ValueFormatter for BytesFormatter { + fn scale_values(&self, typical: f64, values: &mut [f64]) -> &'static str { + scale_bytes(typical, values) + } + + fn scale_throughputs( + &self, + typical: f64, + _throughput: &Throughput, + values: &mut [f64], + ) -> &'static str { + scale_bytes(typical, values) + } + + fn scale_for_machines(&self, values: &mut [f64]) -> &'static str { + // Machine-readable: always bytes + let _ = values; + "B" + } +} + +struct PeakMemory; + +impl Measurement for PeakMemory { + type Intermediate = usize; + type Value = usize; + + fn start(&self) -> Self::Intermediate { + reset_peak(); + live_bytes() + } + + fn end(&self, baseline: Self::Intermediate) -> Self::Value { + peak_bytes().saturating_sub(baseline) + } + + fn add(&self, v1: &Self::Value, v2: &Self::Value) -> Self::Value { + *v1 + *v2 + } + + fn zero(&self) -> Self::Value { + 0 + } + + fn to_f64(&self, value: &Self::Value) -> f64 { + *value as f64 + } + + fn formatter(&self) -> &dyn ValueFormatter { + &BytesFormatter + } +} + +struct AllocatedBytes; + +impl Measurement for AllocatedBytes { + type Intermediate = (); + type Value = usize; + + fn start(&self) -> Self::Intermediate { + reset_allocated(); + } + + fn end(&self, _baseline: Self::Intermediate) -> Self::Value { + allocated_bytes() + } + + fn add(&self, v1: &Self::Value, v2: &Self::Value) -> Self::Value { + *v1 + *v2 + } + + fn zero(&self) -> Self::Value { + 0 + } + + fn to_f64(&self, value: &Self::Value) -> f64 { + *value as f64 + } + + fn formatter(&self) -> &dyn ValueFormatter { + &BytesFormatter + } +} + +// --------------------------------------------------------------------------- +// Test data generation +// --------------------------------------------------------------------------- + +const NUM_ROW_GROUPS: usize = 2; +const PAGES_PER_GROUP: usize = 4; +const VALUES_PER_PAGE: usize = 10_000; +const MAX_LIST_LEN: usize = 10; +const BATCH_SIZE: usize = 8192; +const EXPECTED_VALUE_COUNT: usize = NUM_ROW_GROUPS * PAGES_PER_GROUP * VALUES_PER_PAGE; + +fn seedable_rng() -> StdRng { + StdRng::seed_from_u64(42) +} + +fn build_int32_list_schema() -> (SchemaDescriptor, ColumnDescPtr) { + let message_type = " + message schema { + OPTIONAL GROUP int32_list (LIST) { + repeated group list { + optional INT32 element; + } + } + } + "; + let schema = parse_message_type(message_type) + .map(|t| SchemaDescriptor::new(Arc::new(t))) + .unwrap(); + let col_desc = schema.column(0); + (schema, col_desc) +} + +fn build_string_list_schema() -> (SchemaDescriptor, ColumnDescPtr) { + let message_type = " + message schema { + OPTIONAL GROUP string_list (LIST) { + repeated group list { + optional BYTE_ARRAY element (UTF8); + } + } + } + "; + let schema = parse_message_type(message_type) + .map(|t| SchemaDescriptor::new(Arc::new(t))) + .unwrap(); + let col_desc = schema.column(0); + (schema, col_desc) +} + +const FIXED_BYTE_LEN: usize = 32; + +fn build_fixed32_list_schema() -> (SchemaDescriptor, ColumnDescPtr) { + let message_type = " + message schema { + OPTIONAL GROUP fixed32_list (LIST) { + repeated group list { + optional FIXED_LEN_BYTE_ARRAY(32) element; + } + } + } + "; + let schema = parse_message_type(message_type) + .map(|t| SchemaDescriptor::new(Arc::new(t))) + .unwrap(); + let col_desc = schema.column(0); + (schema, col_desc) +} + +fn build_double_list_schema() -> (SchemaDescriptor, ColumnDescPtr) { + let message_type = " + message schema { + OPTIONAL GROUP double_list (LIST) { + repeated group list { + optional DOUBLE element; + } + } + } + "; + let schema = parse_message_type(message_type) + .map(|t| SchemaDescriptor::new(Arc::new(t))) + .unwrap(); + let col_desc = schema.column(0); + (schema, col_desc) +} + +fn build_list_pages( + column_desc: ColumnDescPtr, + null_density: f32, + mut gen_value: impl FnMut(&mut StdRng) -> T::T, +) -> impl PageIterator + Clone { + let max_def_level = column_desc.max_def_level(); + let max_rep_level = column_desc.max_rep_level(); + let mut rng = seedable_rng(); + let mut pages: Vec> = Vec::new(); + + for _ in 0..NUM_ROW_GROUPS { + let mut column_chunk_pages = Vec::new(); + for _ in 0..PAGES_PER_GROUP { + let mut values = Vec::new(); + let mut def_levels = Vec::new(); + let mut rep_levels = Vec::new(); + + for _ in 0..VALUES_PER_PAGE { + rep_levels.push(0); + if rng.random::() < null_density { + def_levels.push(0); + continue; + } + let len = rng.random_range(0..MAX_LIST_LEN); + if len == 0 { + def_levels.push(1); + continue; + } + (1..len).for_each(|_| rep_levels.push(1)); + for _ in 0..len { + if rng.random::() < null_density { + def_levels.push(2); + } else { + def_levels.push(3); + values.push(gen_value(&mut rng)); + } + } + } + + let mut pb = DataPageBuilderImpl::new(column_desc.clone(), values.len() as u32, true); + pb.add_rep_levels(max_rep_level, &rep_levels); + pb.add_def_levels(max_def_level, &def_levels); + pb.add_values::(Encoding::PLAIN, &values); + column_chunk_pages.push(pb.consume()); + } + pages.push(column_chunk_pages); + } + InMemoryPageIterator::new(pages) +} + +fn drain_reader(mut reader: Box) -> usize { + let mut total_count = 0; + loop { + let batch = reader.next_batch(BATCH_SIZE).unwrap(); + let batch_len = batch.len(); + total_count += batch_len; + if batch_len < BATCH_SIZE { + break; + } + } + total_count +} + +// --------------------------------------------------------------------------- +// Benchmarks +// --------------------------------------------------------------------------- + +fn null_density_label(null_density: f32) -> &'static str { + match (null_density * 100.0) as u32 { + 0 => "no NULLs", + 50 => "half NULLs", + 90 => "90pct NULLs", + 99 => "99pct NULLs", + _ => unreachable!(), + } +} + +fn bench_list_memory( + group: &mut BenchmarkGroup, + null_density: f32, + pages: impl PageIterator + Clone + 'static, + make_reader: fn(Box, ColumnDescPtr) -> Box, + column_desc: &ColumnDescPtr, +) { + group.bench_function(null_density_label(null_density), |b| { + b.iter_batched( + || make_reader(Box::new(pages.clone()), column_desc.clone()), + |reader| { + let count = drain_reader(reader); + assert_eq!(count, EXPECTED_VALUE_COUNT); + count + }, + criterion::BatchSize::PerIteration, + ); + }); +} + +fn int32_list_wrapper( + pages: Box, + column_desc: ColumnDescPtr, +) -> Box { + let child: Box = Box::new( + PrimitiveArrayReader::::new(pages, column_desc, None, DEFAULT_BATCH_SIZE) + .unwrap(), + ); + let field = Field::new_list_field(DataType::Int32, true); + let data_type = DataType::List(Arc::new(field)); + Box::new(ListArrayReader::::new(child, data_type, 2, 1, true)) +} + +fn double_list_wrapper( + pages: Box, + column_desc: ColumnDescPtr, +) -> Box { + let child: Box = Box::new( + PrimitiveArrayReader::::new(pages, column_desc, None, DEFAULT_BATCH_SIZE) + .unwrap(), + ); + let field = Field::new_list_field(DataType::Float64, true); + let data_type = DataType::List(Arc::new(field)); + Box::new(ListArrayReader::::new(child, data_type, 2, 1, true)) +} + +fn fixed32_list_wrapper( + pages: Box, + column_desc: ColumnDescPtr, +) -> Box { + let child = + make_fixed_len_byte_array_reader(pages, column_desc, None, DEFAULT_BATCH_SIZE).unwrap(); + let field = Field::new_list_field(DataType::FixedSizeBinary(FIXED_BYTE_LEN as i32), true); + let data_type = DataType::List(Arc::new(field)); + Box::new(ListArrayReader::::new(child, data_type, 2, 1, true)) +} + +fn string_list_wrapper( + pages: Box, + column_desc: ColumnDescPtr, +) -> Box { + let child = make_byte_array_reader(pages, column_desc, None, DEFAULT_BATCH_SIZE).unwrap(); + let field = Field::new_list_field(DataType::Utf8, true); + let data_type = DataType::List(Arc::new(field)); + Box::new(ListArrayReader::::new(child, data_type, 2, 1, true)) +} + +fn add_benches(c: &mut Criterion, measurement_name: &str) { + let (_schema, int32_desc) = build_int32_list_schema(); + let (_schema, double_desc) = build_double_list_schema(); + let (_schema, fixed32_desc) = build_fixed32_list_schema(); + let (_schema, string_desc) = build_string_list_schema(); + + let mut group = c.benchmark_group(format!( + "arrow_array_reader/ListArray_{measurement_name}/Int32List" + )); + for null_density in [0.0, 0.5, 0.9, 0.99] { + let pages = + build_list_pages::(int32_desc.clone(), null_density, |rng| rng.random()); + bench_list_memory( + &mut group, + null_density, + pages, + int32_list_wrapper, + &int32_desc, + ); + } + group.finish(); + + let mut group = c.benchmark_group(format!( + "arrow_array_reader/ListArray_{measurement_name}/DoubleList" + )); + for null_density in [0.0, 0.5, 0.9, 0.99] { + let pages = + build_list_pages::(double_desc.clone(), null_density, |rng| rng.random()); + bench_list_memory( + &mut group, + null_density, + pages, + double_list_wrapper, + &double_desc, + ); + } + group.finish(); + + let mut group = c.benchmark_group(format!( + "arrow_array_reader/ListArray_{measurement_name}/Fixed32List" + )); + for null_density in [0.0, 0.5, 0.9, 0.99] { + let pages = + build_list_pages::(fixed32_desc.clone(), null_density, |rng| { + let mut buf = vec![0u8; FIXED_BYTE_LEN]; + rng.fill(&mut buf[..]); + buf.into() + }); + bench_list_memory( + &mut group, + null_density, + pages, + fixed32_list_wrapper, + &fixed32_desc, + ); + } + group.finish(); + + let mut group = c.benchmark_group(format!( + "arrow_array_reader/ListArray_{measurement_name}/StringList" + )); + for null_density in [0.0, 0.5, 0.9, 0.99] { + let pages = build_list_pages::(string_desc.clone(), null_density, |rng| { + let len = rng.random_range(5..50); + let bytes: Vec = (0..len).map(|_| rng.random_range(b'a'..=b'z')).collect(); + bytes.into() + }); + bench_list_memory( + &mut group, + null_density, + pages, + string_list_wrapper, + &string_desc, + ); + } + group.finish(); +} + +fn add_peak_memory_benches(c: &mut Criterion) { + add_benches(c, "peak_memory"); +} + +fn add_allocated_bytes_benches(c: &mut Criterion) { + add_benches(c, "allocated_bytes"); +} + +fn peak_memory_criterion() -> Criterion { + Criterion::default() + .with_measurement(PeakMemory) + .sample_size(10) +} + +fn allocated_bytes_criterion() -> Criterion { + Criterion::default() + .with_measurement(AllocatedBytes) + .sample_size(10) +} + +criterion_group! { + name = peak_memory; + config = peak_memory_criterion(); + targets = add_peak_memory_benches +} + +criterion_group! { + name = cumulative_allocated_bytes; + config = allocated_bytes_criterion(); + targets = add_allocated_bytes_benches +} +criterion_main!(peak_memory, cumulative_allocated_bytes); From 48fa8a7a45567b9ab47c461771b968ba0d37812f Mon Sep 17 00:00:00 2001 From: Hippolyte Barraud Date: Tue, 12 May 2026 11:03:34 -0400 Subject: [PATCH 30/33] feat(parquet): separate push decoder frontier state from row-group decoding (#9804) # Which issue does this PR close? - Prerequisite to #9697 # Rationale for this change #9697 aims to make staged buffer management in the push decoder more explicit. In doing so, it exposes a structural problem: the logic for deciding whether a row group is still live, skipped, or unreachable is spread across several parts of the decoder. This matters because row-group-level buffer release depends on a single question having a clear answer: can this row group ever need bytes again? That answer depends on the queued row groups, the remaining selection, the running offset/limit budget, and whether predicates require the decoder to stay conservative. Today, that state is split across multiple components, which makes the release policy difficult to centralize cleanly. # What changes are included in this PR? This PR introduces a clearer ownership boundary in the push decoder: - cross-row-group scan state is now handled by a dedicated frontier/look-ahead mechanism - the row-group builder is reduced to current-row-group decode work only - offset/limit accounting and row-group selection advancement are centralized around that frontier/builder split This does not implement row-group-level buffer release directly, but it establishes the structure needed for that follow-up work. It should also make future pruning rules easier to add and maintain. # Are these changes tested? All existing tests pass, and the refactor adds focused coverage for the extracted budget logic and the frontier-driven `try_next_reader` path. # Are there any user-facing changes? None. --------- Signed-off-by: Hippolyte Barraud --- parquet/src/arrow/push_decoder/mod.rs | 31 +- .../arrow/push_decoder/reader_builder/mod.rs | 279 +++++++++++++----- parquet/src/arrow/push_decoder/remaining.rs | 243 ++++++++++++--- 3 files changed, 433 insertions(+), 120 deletions(-) diff --git a/parquet/src/arrow/push_decoder/mod.rs b/parquet/src/arrow/push_decoder/mod.rs index 4c667e534366..f905d6fb2ccb 100644 --- a/parquet/src/arrow/push_decoder/mod.rs +++ b/parquet/src/arrow/push_decoder/mod.rs @@ -30,7 +30,7 @@ use crate::file::metadata::ParquetMetaData; use crate::util::push_buffers::PushBuffers; use arrow_array::RecordBatch; use bytes::Bytes; -use reader_builder::RowGroupReaderBuilder; +use reader_builder::{RowBudget, RowGroupReaderBuilder}; use remaining::RemainingRowGroups; use std::ops::Range; use std::sync::Arc; @@ -181,6 +181,9 @@ impl ParquetPushDecoderBuilder { // If no row groups were specified, read all of them let row_groups = row_groups.unwrap_or_else(|| (0..parquet_metadata.num_row_groups()).collect()); + let has_predicates = filter + .as_ref() + .is_some_and(|filter| !filter.predicates.is_empty()); // Prepare to build RowGroup readers let file_len = 0; // not used in push decoder @@ -191,8 +194,6 @@ impl ParquetPushDecoderBuilder { Arc::clone(&parquet_metadata), fields, filter, - limit, - offset, metrics, max_predicate_cache_size, buffers, @@ -204,6 +205,8 @@ impl ParquetPushDecoderBuilder { parquet_metadata, row_groups, selection, + RowBudget::new(offset, limit), + has_predicates, row_group_reader_builder, ); @@ -1402,6 +1405,28 @@ mod test { expect_finished(decoder.try_decode()); } + #[test] + fn test_decoder_try_next_reader_offset_limit() { + let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()) + .unwrap() + .with_offset(225) + .with_limit(20) + .build() + .unwrap(); + + let ranges = expect_needs_data(decoder.try_next_reader()); + push_ranges_to_decoder(&mut decoder, ranges); + + let reader = expect_data(decoder.try_next_reader()); + let batches = reader + .map(|batch| batch.expect("expected decoded batch")) + .collect::>(); + let output = concat_batches(&TEST_BATCH.schema(), &batches).unwrap(); + assert_eq!(output, TEST_BATCH.slice(225, 20)); + + expect_finished(decoder.try_next_reader()); + } + #[test] fn test_decoder_row_group_selection() { // take only the second row group diff --git a/parquet/src/arrow/push_decoder/reader_builder/mod.rs b/parquet/src/arrow/push_decoder/reader_builder/mod.rs index 60e50d29524e..0452cea436a3 100644 --- a/parquet/src/arrow/push_decoder/reader_builder/mod.rs +++ b/parquet/src/arrow/push_decoder/reader_builder/mod.rs @@ -18,7 +18,6 @@ mod data; mod filter; -use crate::DecodeResult; use crate::arrow::ProjectionMask; use crate::arrow::array_reader::{ArrayReaderBuilder, CacheOptions, RowGroupCache}; use crate::arrow::arrow_reader::metrics::ArrowReaderMetrics; @@ -42,12 +41,13 @@ use filter::FilterInfo; use std::ops::Range; use std::sync::{Arc, RwLock}; -/// The current row group being read and the read plan +/// The current row group being read, its read plan, and its offset/limit budget. #[derive(Debug)] struct RowGroupInfo { row_group_idx: usize, row_count: usize, plan_builder: ReadPlanBuilder, + budget: RowBudget, } /// This is the inner state machine for reading a single row group. @@ -88,6 +88,107 @@ enum RowGroupDecoderState { Finished, } +/// Running offset/limit budget shared across row groups. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(crate) struct RowBudget { + offset: Option, + limit: Option, +} + +impl RowBudget { + pub(crate) fn new(offset: Option, limit: Option) -> Self { + Self { offset, limit } + } + + pub(crate) fn is_exhausted(self) -> bool { + matches!(self.limit, Some(0)) + } + + /// Returns how many selected rows remain after applying this budget. + pub(crate) fn rows_after(self, rows_before_budget: usize) -> usize { + let rows_after_offset = rows_before_budget.saturating_sub(self.offset.unwrap_or(0)); + match self.limit { + Some(limit) => rows_after_offset.min(limit), + None => rows_after_offset, + } + } + + /// Returns the number of selected rows needed before applying the offset. + fn selected_row_limit(self) -> Option { + self.limit + .map(|limit| limit.saturating_add(self.offset.unwrap_or(0))) + } + + fn apply_to_plan(self, plan_builder: ReadPlanBuilder, row_count: usize) -> BudgetedReadPlan { + let rows_before_budget = plan_builder.num_rows_selected().unwrap_or(row_count); + let plan_builder = plan_builder + .limited(row_count) + .with_offset(self.offset) + .with_limit(self.limit) + .build_limited(); + let rows_after_budget = self.rows_after(rows_before_budget); + + BudgetedReadPlan { + plan_builder, + rows_before_budget, + rows_after_budget, + remaining_budget: self.advance(rows_before_budget, rows_after_budget), + } + } + + /// Advance the budget past one row group. + /// + /// `rows_before_budget` is the number of rows selected before applying the + /// budget, and `rows_after_budget` is the number retained for output from + /// this row group. + pub(crate) fn advance(mut self, rows_before_budget: usize, rows_after_budget: usize) -> Self { + if let Some(offset) = &mut self.offset { + // Reduction is either because of offset or limit, as limit is applied + // after offset has been "exhausted" can just use saturating sub here. + *offset = offset.saturating_sub(rows_before_budget - rows_after_budget); + } + + if rows_after_budget != 0 { + if let Some(limit) = &mut self.limit { + *limit -= rows_after_budget; + } + } + + self + } +} + +#[derive(Debug)] +struct BudgetedReadPlan { + /// Read plan after applying this row group's share of the offset/limit budget. + plan_builder: ReadPlanBuilder, + /// Number of rows selected by row selection and predicates before applying + /// this row group's offset/limit budget. + rows_before_budget: usize, + /// Number of selected rows that remain to be read after applying this row + /// group's offset/limit budget. + rows_after_budget: usize, + /// Budget remaining for later row groups. + remaining_budget: RowBudget, +} + +#[derive(Debug)] +pub(crate) enum RowGroupBuildResult { + /// The active row group is complete without producing a reader. + Finished { + /// Budget remaining after applying this row group's selection. + remaining_budget: RowBudget, + }, + /// More bytes are needed before the active row group can make progress. + NeedsData(Vec>), + /// The active row group produced a reader. + Data { + batch_reader: ParquetRecordBatchReader, + /// Budget remaining after applying this row group's selection. + remaining_budget: RowBudget, + }, +} + /// Result of a state transition #[derive(Debug)] struct NextState { @@ -96,7 +197,7 @@ struct NextState { /// /// * `Some`: the processing should stop and return the result /// * `None`: processing should continue - result: Option>, + result: Option, } impl NextState { @@ -111,10 +212,7 @@ impl NextState { } /// Create a NextState with a result that should be returned - fn result( - next_state: RowGroupDecoderState, - result: DecodeResult, - ) -> Self { + fn result(next_state: RowGroupDecoderState, result: RowGroupBuildResult) -> Self { Self { next_state, result: Some(result), @@ -144,12 +242,6 @@ pub(crate) struct RowGroupReaderBuilder { /// Optional filter filter: Option, - /// Limit to apply to remaining row groups (decremented as rows are read) - limit: Option, - - /// Offset to apply to remaining row groups (decremented as rows are read) - offset: Option, - /// The size in bytes of the predicate cache to use /// /// See [`RowGroupCache`] for details. @@ -180,8 +272,6 @@ impl RowGroupReaderBuilder { metadata: Arc, fields: Option>, filter: Option, - limit: Option, - offset: Option, metrics: ArrowReaderMetrics, max_predicate_cache_size: usize, buffers: PushBuffers, @@ -193,8 +283,6 @@ impl RowGroupReaderBuilder { metadata, fields, filter, - limit, - offset, metrics, max_predicate_cache_size, row_selection_policy, @@ -233,12 +321,18 @@ impl RowGroupReaderBuilder { }) } + /// Returns true if this builder is currently decoding a row group. + pub(crate) fn has_active_row_group(&self) -> bool { + !matches!(self.state, Some(RowGroupDecoderState::Finished)) + } + /// Setup this reader to read the next row group pub(crate) fn next_row_group( &mut self, row_group_idx: usize, row_count: usize, selection: Option, + budget: RowBudget, ) -> Result<(), ParquetError> { let state = self.take_state()?; if !matches!(state, RowGroupDecoderState::Finished) { @@ -254,22 +348,20 @@ impl RowGroupReaderBuilder { row_group_idx, row_count, plan_builder, + budget, }; self.state = Some(RowGroupDecoderState::Start { row_group_info }); Ok(()) } - /// Try to build the next `ParquetRecordBatchReader` from this RowGroupReader. + /// Try to build the next `ParquetRecordBatchReader` for the active row group. /// - /// If more data is needed, returns [`DecodeResult::NeedsData`] with the - /// ranges of data that are needed to proceed. - /// - /// If a [`ParquetRecordBatchReader`] is ready, it is returned in - /// `DecodeResult::Data`. - pub(crate) fn try_build( - &mut self, - ) -> Result, ParquetError> { + /// Returns [`RowGroupBuildResult::NeedsData`] if more data is needed, + /// [`RowGroupBuildResult::Data`] if a reader is ready, or + /// [`RowGroupBuildResult::Finished`] if the row group completed without + /// producing a reader. + pub(crate) fn try_build(&mut self) -> Result { loop { let current_state = self.take_state()?; // Try to transition the decoder. @@ -310,18 +402,10 @@ impl RowGroupReaderBuilder { ) -> Result { let result = match current_state { RowGroupDecoderState::Start { row_group_info } => { - // Short-circuit once the overall output limit is exhausted. - // - // `self.limit` tracks how many more rows the reader is still - // allowed to emit and is decremented as each row group is - // planned in `StartData`, so `Some(0)` means earlier row - // groups have already produced the full requested output. - if matches!(self.limit, Some(0)) { - return Ok(NextState::result( - RowGroupDecoderState::Finished, - DecodeResult::Finished, - )); - } + debug_assert!( + !row_group_info.budget.is_exhausted(), + "RowGroupFrontier should not hand off row groups after the output limit is exhausted" + ); let column_chunks = None; // no prior column chunks @@ -371,6 +455,7 @@ impl RowGroupReaderBuilder { row_group_idx, row_count, plan_builder, + budget, } = row_group_info; // If nothing is selected, we are done with this row group @@ -379,7 +464,9 @@ impl RowGroupReaderBuilder { self.filter = Some(filter_info.into_filter()); return Ok(NextState::result( RowGroupDecoderState::Finished, - DecodeResult::Finished, + RowGroupBuildResult::Finished { + remaining_budget: budget, + }, )); } @@ -405,6 +492,7 @@ impl RowGroupReaderBuilder { row_group_idx, row_count, plan_builder, + budget, }; NextState::again(RowGroupDecoderState::WaitingOnFilterData { @@ -428,7 +516,7 @@ impl RowGroupReaderBuilder { filter_info, data_request, }, - DecodeResult::NeedsData(needed_ranges), + RowGroupBuildResult::NeedsData(needed_ranges), )); } @@ -437,6 +525,7 @@ impl RowGroupReaderBuilder { row_group_idx, row_count, mut plan_builder, + budget, } = row_group_info; let predicate = filter_info.current(); @@ -476,10 +565,10 @@ impl RowGroupReaderBuilder { // When this is the final predicate in the chain and an output // limit is set, tell the filter evaluation to stop once enough // matching rows have been accumulated. - let predicate_limit = self - .limit - .filter(|_| filter_info.is_last()) - .map(|l| l.saturating_add(self.offset.unwrap_or(0))); + let predicate_limit = filter_info + .is_last() + .then(|| budget.selected_row_limit()) + .flatten(); // Evaluate the filter via `with_predicate_options`, opting into // early termination when this is the final predicate and an @@ -495,6 +584,7 @@ impl RowGroupReaderBuilder { row_group_idx, row_count, plan_builder, + budget, }; // Take back the column chunks that were read @@ -531,47 +621,32 @@ impl RowGroupReaderBuilder { row_group_idx, row_count, plan_builder, + budget, } = row_group_info; - // Compute the number of rows in the selection before applying limit and offset - let rows_before = plan_builder.num_rows_selected().unwrap_or(row_count); + let BudgetedReadPlan { + mut plan_builder, + rows_before_budget, + rows_after_budget, + remaining_budget, + } = budget.apply_to_plan(plan_builder, row_count); - if rows_before == 0 { + if rows_before_budget == 0 { // ruled out entire row group return Ok(NextState::result( RowGroupDecoderState::Finished, - DecodeResult::Finished, + RowGroupBuildResult::Finished { remaining_budget }, )); } - // Apply any limit and offset - let mut plan_builder = plan_builder - .limited(row_count) - .with_offset(self.offset) - .with_limit(self.limit) - .build_limited(); - - let rows_after = plan_builder.num_rows_selected().unwrap_or(row_count); - - // Update running offset and limit for after the current row group is read - if let Some(offset) = &mut self.offset { - // Reduction is either because of offset or limit, as limit is applied - // after offset has been "exhausted" can just use saturating sub here - *offset = offset.saturating_sub(rows_before - rows_after) - } - - if rows_after == 0 { + if rows_after_budget == 0 { // no rows left after applying limit/offset return Ok(NextState::result( RowGroupDecoderState::Finished, - DecodeResult::Finished, + RowGroupBuildResult::Finished { remaining_budget }, )); } - if let Some(limit) = &mut self.limit { - *limit -= rows_after; - } - let data_request = DataRequestBuilder::new( row_group_idx, row_count, @@ -597,6 +672,7 @@ impl RowGroupReaderBuilder { row_group_idx, row_count, plan_builder, + budget: remaining_budget, }; NextState::again(RowGroupDecoderState::WaitingOnData { @@ -620,7 +696,7 @@ impl RowGroupReaderBuilder { data_request, cache_info, }, - DecodeResult::NeedsData(needed_ranges), + RowGroupBuildResult::NeedsData(needed_ranges), )); } @@ -629,6 +705,7 @@ impl RowGroupReaderBuilder { row_group_idx, row_count, plan_builder, + budget, } = row_group_info; let row_group = data_request.try_into_in_memory_row_group( @@ -656,11 +733,18 @@ impl RowGroupReaderBuilder { }?; let reader = ParquetRecordBatchReader::new(array_reader, plan); - NextState::result(RowGroupDecoderState::Finished, DecodeResult::Data(reader)) + NextState::result( + RowGroupDecoderState::Finished, + RowGroupBuildResult::Data { + batch_reader: reader, + remaining_budget: budget, + }, + ) } RowGroupDecoderState::Finished => { - // nothing left to read - NextState::result(RowGroupDecoderState::Finished, DecodeResult::Finished) + return Err(ParquetError::General(String::from( + "Internal Error: try_build called without an active row group", + ))); } }; Ok(result) @@ -760,10 +844,55 @@ fn override_selector_strategy_if_needed( #[cfg(test)] mod tests { use super::*; + use crate::arrow::arrow_reader::{RowSelection, RowSelector}; #[test] // Verify that the size of RowGroupDecoderState does not grow too large fn test_structure_size() { - assert_eq!(std::mem::size_of::(), 200); + assert_eq!(std::mem::size_of::(), 232); + } + + #[test] + fn test_row_budget_offset_limit_across_row_groups() { + let first = + RowBudget::new(Some(225), Some(20)).apply_to_plan(ReadPlanBuilder::new(1024), 200); + assert_eq!(first.rows_before_budget, 200); + assert_eq!(first.rows_after_budget, 0); + assert_eq!(first.remaining_budget, RowBudget::new(Some(25), Some(20))); + assert_eq!(first.plan_builder.num_rows_selected(), Some(0)); + + let second = first + .remaining_budget + .apply_to_plan(ReadPlanBuilder::new(1024), 200); + assert_eq!(second.rows_before_budget, 200); + assert_eq!(second.rows_after_budget, 20); + assert_eq!(second.remaining_budget, RowBudget::new(Some(0), Some(0))); + assert_eq!(second.plan_builder.num_rows_selected(), Some(20)); + } + + #[test] + fn test_row_budget_limit_only() { + let budgeted = + RowBudget::new(None, Some(20)).apply_to_plan(ReadPlanBuilder::new(1024), 200); + assert_eq!(budgeted.rows_before_budget, 200); + assert_eq!(budgeted.rows_after_budget, 20); + assert_eq!(budgeted.remaining_budget, RowBudget::new(None, Some(0))); + assert_eq!(budgeted.plan_builder.num_rows_selected(), Some(20)); + } + + #[test] + fn test_row_budget_empty_selection() { + let empty_selection = RowSelection::from(vec![RowSelector::skip(200)]); + let budgeted = RowBudget::new(Some(10), Some(20)).apply_to_plan( + ReadPlanBuilder::new(1024).with_selection(Some(empty_selection)), + 200, + ); + assert_eq!(budgeted.rows_before_budget, 0); + assert_eq!(budgeted.rows_after_budget, 0); + assert_eq!( + budgeted.remaining_budget, + RowBudget::new(Some(10), Some(20)) + ); + assert_eq!(budgeted.plan_builder.num_rows_selected(), Some(0)); } } diff --git a/parquet/src/arrow/push_decoder/remaining.rs b/parquet/src/arrow/push_decoder/remaining.rs index 2986ca0da8d8..33e13abf9c12 100644 --- a/parquet/src/arrow/push_decoder/remaining.rs +++ b/parquet/src/arrow/push_decoder/remaining.rs @@ -17,7 +17,9 @@ use crate::DecodeResult; use crate::arrow::arrow_reader::{ParquetRecordBatchReader, RowSelection}; -use crate::arrow::push_decoder::reader_builder::RowGroupReaderBuilder; +use crate::arrow::push_decoder::reader_builder::{ + RowBudget, RowGroupBuildResult, RowGroupReaderBuilder, +}; use crate::errors::ParquetError; use crate::file::metadata::ParquetMetaData; use bytes::Bytes; @@ -25,21 +27,166 @@ use std::collections::VecDeque; use std::ops::Range; use std::sync::Arc; -/// State machine that tracks the remaining high level chunks (row groups) of -/// Parquet data are left to read. -/// -/// This is currently a row group, but the author aspires to extend the pattern -/// to data boundaries other than RowGroups in the future. +/// Plan for the next queued row group after row-selection slicing. #[derive(Debug)] -pub(crate) struct RemainingRowGroups { - /// The underlying Parquet metadata - parquet_metadata: Arc, +enum QueuedRowGroupDecision { + /// Hand this row group to the builder. + Read(NextRowGroup), + /// Skip this row group, and keep scanning with the updated budget. + Skip { remaining_budget: RowBudget }, +} - /// The row groups that have not yet been read - row_groups: VecDeque, +/// Work item handed from [`RowGroupFrontier`] to [`RowGroupReaderBuilder`]. +#[derive(Debug)] +struct NextRowGroup { + row_group_idx: usize, + row_count: usize, + /// This row group's slice of the global selection, or `None` when all rows + /// are selected. + selection: Option, + /// Budget snapshot to apply while decoding this row group. + budget: RowBudget, +} - /// Remaining selection to apply to the next row groups +#[derive(Debug)] +struct RowGroupFrontier { + /// Metadata used to resolve row counts for queued row groups. + parquet_metadata: Arc, + /// Row group indices not yet handed to the builder. + row_groups: VecDeque, + /// Cross-row-group cursor for the optional global row selection. selection: Option, + /// Offset/limit budget before the next readable row group is planned. + budget: RowBudget, + /// If predicates are present, row groups with selected rows must be read so + /// the predicate can decide whether they are actually needed. + has_predicates: bool, +} + +impl RowGroupFrontier { + fn new( + parquet_metadata: Arc, + row_groups: Vec, + selection: Option, + budget: RowBudget, + has_predicates: bool, + ) -> Self { + Self { + parquet_metadata, + row_groups: VecDeque::from(row_groups), + selection, + budget, + has_predicates, + } + } + + fn row_group_num_rows(&self, row_group_idx: usize) -> Result { + self.parquet_metadata + .row_group(row_group_idx) + .num_rows() + .try_into() + .map_err(|e| ParquetError::General(format!("Row count overflow: {e}"))) + } + + fn update_budget_after_row_group(&mut self, budget: RowBudget) { + self.budget = budget; + } + + fn clear_remaining(&mut self) { + self.selection = None; + self.row_groups.clear(); + } + + /// Plan whether a selected row group should be read or skipped. + /// + /// Selection-only skips are handled before this method is called. This + /// method applies the remaining offset/limit budget and predicate + /// conservatism. + fn plan_selected_row_group( + &self, + next_row_group: NextRowGroup, + selected_rows: usize, + ) -> QueuedRowGroupDecision { + if self.has_predicates { + return QueuedRowGroupDecision::Read(next_row_group); + } + + let rows_after_budget = self.budget.rows_after(selected_rows); + if rows_after_budget != 0 { + return QueuedRowGroupDecision::Read(next_row_group); + } + + QueuedRowGroupDecision::Skip { + remaining_budget: self.budget.advance(selected_rows, rows_after_budget), + } + } + + /// Advance queued row groups until one should be handed to the builder. + fn next_readable_row_group(&mut self) -> Result, ParquetError> { + loop { + let Some(&row_group_idx) = self.row_groups.front() else { + return Ok(None); + }; + if self.budget.is_exhausted() + || self + .selection + .as_ref() + .is_some_and(|selection| selection.row_count() == 0) + { + self.clear_remaining(); + return Ok(None); + } + + let row_count = self.row_group_num_rows(row_group_idx)?; + let (selection, selected_rows) = match self.selection.as_mut() { + Some(selection) => { + let selection = selection.split_off(row_count); + let selected_rows = selection.row_count(); + if selected_rows == 0 { + self.row_groups.pop_front(); + continue; + } + + let selection = if selected_rows == row_count { + None + } else { + Some(selection) + }; + (selection, selected_rows) + } + None => (None, row_count), + }; + + let next_row_group = NextRowGroup { + row_group_idx, + row_count, + selection, + budget: self.budget, + }; + + match self.plan_selected_row_group(next_row_group, selected_rows) { + QueuedRowGroupDecision::Read(next_row_group) => { + self.row_groups.pop_front(); + return Ok(Some(next_row_group)); + } + QueuedRowGroupDecision::Skip { remaining_budget } => { + self.row_groups.pop_front(); + self.budget = remaining_budget; + } + } + } + } +} + +/// State machine that tracks the remaining high level chunks (row groups) of +/// Parquet data left to read. +/// +/// [`RowGroupFrontier`] owns cross-row-group scan state and selects the next +/// work item. [`RowGroupReaderBuilder`] owns decoding for the active row group. +#[derive(Debug)] +pub(crate) struct RemainingRowGroups { + /// Cross-row-group scan state for queued work. + frontier: RowGroupFrontier, /// State for building the reader for the current row group row_group_reader_builder: RowGroupReaderBuilder, @@ -50,12 +197,18 @@ impl RemainingRowGroups { parquet_metadata: Arc, row_groups: Vec, selection: Option, + budget: RowBudget, + has_predicates: bool, row_group_reader_builder: RowGroupReaderBuilder, ) -> Self { Self { - parquet_metadata, - row_groups: VecDeque::from(row_groups), - selection, + frontier: RowGroupFrontier::new( + parquet_metadata, + row_groups, + selection, + budget, + has_predicates, + ), row_group_reader_builder, } } @@ -82,42 +235,48 @@ impl RemainingRowGroups { &mut self, ) -> Result, ParquetError> { loop { - // Are we ready yet to start reading? - let result: DecodeResult = - self.row_group_reader_builder.try_build()?; - match result { - DecodeResult::Finished => { + if !self.row_group_reader_builder.has_active_row_group() { + // We are done with the previous row group, seek to the next one + // from the frontier, if any. + + match self.frontier.next_readable_row_group()? { + Some(NextRowGroup { + row_group_idx, + row_count, + selection, + budget, + }) => { + self.row_group_reader_builder.next_row_group( + row_group_idx, + row_count, + selection, + budget, + )?; + } + None => return Ok(DecodeResult::Finished), + } + } + + match self.row_group_reader_builder.try_build()? { + RowGroupBuildResult::Finished { remaining_budget } => { + self.frontier + .update_budget_after_row_group(remaining_budget); // reader is done, proceed to the next row group - // fall through to the next row group - // This happens if the row group was completely filtered out } - DecodeResult::NeedsData(ranges) => { + RowGroupBuildResult::NeedsData(ranges) => { // need more data to proceed return Ok(DecodeResult::NeedsData(ranges)); } - DecodeResult::Data(batch_reader) => { + RowGroupBuildResult::Data { + batch_reader, + remaining_budget, + } => { + self.frontier + .update_budget_after_row_group(remaining_budget); // ready to read the row group return Ok(DecodeResult::Data(batch_reader)); } } - - // No current reader, proceed to the next row group if any - let row_group_idx = match self.row_groups.pop_front() { - None => return Ok(DecodeResult::Finished), - Some(idx) => idx, - }; - - let row_count: usize = self - .parquet_metadata - .row_group(row_group_idx) - .num_rows() - .try_into() - .map_err(|e| ParquetError::General(format!("Row count overflow: {e}")))?; - - let selection = self.selection.as_mut().map(|s| s.split_off(row_count)); - self.row_group_reader_builder - .next_row_group(row_group_idx, row_count, selection)?; - // the next iteration will try to build the reader for the new row group } } } From ab4d277d3f314cc9e70f15001057d9c1db3c90da Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Mon, 27 Apr 2026 16:52:19 -0500 Subject: [PATCH 31/33] feat(parquet): add ParquetPushDecoder::swap_strategy for adaptive scans MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a small surface to the push decoder so callers can swap the RowFilter, ProjectionMask, and/or RowSelectionPolicy at row-group boundaries without rebuilding the decoder: - new pub fn `can_swap_strategy() -> bool` — true between row groups (outer state `ReadingRowGroup`, inner state `Finished`) - new pub fn `swap_strategy(StrategySwap) -> Result<()>` — rejected with `ParquetError::General` when called mid-row-group - new `pub struct StrategySwap` (`#[non_exhaustive]`) with builder methods `with_projection`, `with_filter`, `with_row_selection_policy` - new pub fn `row_groups_remaining() -> usize` for diagnostics `PushBuffers` carries through the swap, so bytes already fetched for columns that survive into the new strategy are reused — only bytes the new strategy needs but that aren't already buffered get requested via `NeedsData`. Adaptive callers should drive the decoder with `try_next_reader` rather than `try_decode`: handing the active reader off transitions the decoder back to `ReadingRowGroup` immediately, giving the caller a clean swap window between two consecutive returns. `try_decode` loops past row-group boundaries internally and is unsuitable for in-flight strategy changes. Tests cover: filter swap between row groups, mid-row-group rejection, swap-while-iterating-handed-off-reader, and projection narrowing that reuses already-buffered bytes. Co-Authored-By: Claude Opus 4.7 (1M context) --- parquet/src/arrow/push_decoder/mod.rs | 328 ++++++++++++++++++ .../arrow/push_decoder/reader_builder/mod.rs | 57 +++ parquet/src/arrow/push_decoder/remaining.rs | 39 ++- 3 files changed, 423 insertions(+), 1 deletion(-) diff --git a/parquet/src/arrow/push_decoder/mod.rs b/parquet/src/arrow/push_decoder/mod.rs index f905d6fb2ccb..e62cf30e9aee 100644 --- a/parquet/src/arrow/push_decoder/mod.rs +++ b/parquet/src/arrow/push_decoder/mod.rs @@ -22,8 +22,10 @@ mod reader_builder; mod remaining; use crate::DecodeResult; +use crate::arrow::ProjectionMask; use crate::arrow::arrow_reader::{ ArrowReaderBuilder, ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReader, + RowFilter, RowSelectionPolicy, }; use crate::errors::ParquetError; use crate::file::metadata::ParquetMetaData; @@ -377,6 +379,102 @@ impl ParquetPushDecoder { pub fn clear_all_ranges(&mut self) { self.state.clear_all_ranges(); } + + /// True iff the decoder is at a row-group boundary and a + /// [`Self::swap_strategy`] call would succeed. + /// + /// A boundary is "between row groups": the previous row group's + /// [`ParquetRecordBatchReader`] has been fully extracted (via + /// [`Self::try_next_reader`]) or fully drained (via [`Self::try_decode`]), + /// and the next row group has not yet been planned. While + /// [`Self::try_decode`] is iterating an active row group's reader this + /// returns `false`; with [`Self::try_next_reader`] there is a clean + /// window between two consecutive returns where this is `true`. + pub fn can_swap_strategy(&self) -> bool { + self.state.can_swap_strategy() + } + + /// Number of row groups left to decode after the one currently in flight. + /// Useful as a "should I bother computing a new strategy?" signal. + pub fn row_groups_remaining(&self) -> usize { + self.state.row_groups_remaining() + } + + /// Replace projection / row filter / row selection policy for subsequent + /// row groups. + /// + /// Returns `Err(ParquetError::General)` when called outside a row-group + /// boundary; check [`Self::can_swap_strategy`] first. + /// + /// The decoder's [`PushBuffers`] are preserved across the swap. Bytes + /// that have already been fetched for columns the new strategy still + /// needs will not be re-requested. Bytes for columns the new strategy + /// no longer needs remain buffered until [`Self::clear_all_ranges`] is + /// called or the decoder is dropped. + /// + /// Limit, offset, batch size, metadata, fields, and predicate-cache size + /// are unchanged by a swap. + /// + /// [`PushBuffers`]: crate::util::push_buffers::PushBuffers + pub fn swap_strategy(&mut self, swap: StrategySwap) -> Result<(), ParquetError> { + self.state.swap_strategy(swap) + } +} + +/// Description of a strategy swap to apply via +/// [`ParquetPushDecoder::swap_strategy`]. +/// +/// Each field is `Option`-wrapped so callers only override what they intend +/// to change. Fields left as `None` carry their previous value through the +/// swap. +/// +/// [`Self::filter`] is doubly-`Option`-wrapped on purpose: the outer +/// `Option` is "do you want to change the filter?", the inner is "set or +/// clear?". `Some(None)` clears any previously-installed filter. +#[derive(Debug, Default)] +#[non_exhaustive] +pub struct StrategySwap { + /// New projection mask. `None` means "leave the projection alone". + pub projection: Option, + /// New row filter. Outer `None` means "leave the filter alone"; + /// `Some(None)` clears the filter; `Some(Some(filter))` installs a new + /// one. + pub filter: Option>, + /// New row selection policy. `None` means "leave the policy alone". + pub row_selection_policy: Option, +} + +impl StrategySwap { + /// Empty swap. No fields will be modified — chain `with_*` setters to + /// configure. + pub fn new() -> Self { + Self::default() + } + + /// Set a new projection mask for subsequent row groups. + pub fn with_projection(mut self, projection: ProjectionMask) -> Self { + self.projection = Some(projection); + self + } + + /// Install a new row filter (or `None` to clear) for subsequent row + /// groups. + pub fn with_filter(mut self, filter: Option) -> Self { + self.filter = Some(filter); + self + } + + /// Set a new row selection policy for subsequent row groups. + pub fn with_row_selection_policy(mut self, policy: RowSelectionPolicy) -> Self { + self.row_selection_policy = Some(policy); + self + } + + /// True iff every field is `None`. A no-op swap returns successfully but + /// has no effect. + pub fn is_empty(&self) -> bool { + self.projection.is_none() && self.filter.is_none() && self.row_selection_policy.is_none() + } } /// Internal state machine for the [`ParquetPushDecoder`] @@ -599,6 +697,75 @@ impl ParquetDecoderState { ParquetDecoderState::Finished => {} } } + + fn can_swap_strategy(&self) -> bool { + match self { + ParquetDecoderState::ReadingRowGroup { + remaining_row_groups, + } => remaining_row_groups.is_at_row_group_boundary(), + // Mid-row-group: the active reader holds an `ArrayReader` and + // `ReadPlan` keyed to the *current* projection/filter; swapping + // would require throwing that work away. + ParquetDecoderState::DecodingRowGroup { .. } => false, + ParquetDecoderState::Finished => false, + } + } + + fn row_groups_remaining(&self) -> usize { + match self { + ParquetDecoderState::ReadingRowGroup { + remaining_row_groups, + } => remaining_row_groups.row_groups_remaining(), + ParquetDecoderState::DecodingRowGroup { + remaining_row_groups, + .. + } => remaining_row_groups.row_groups_remaining(), + ParquetDecoderState::Finished => 0, + } + } + + fn swap_strategy(&mut self, swap: StrategySwap) -> Result<(), ParquetError> { + if swap.is_empty() { + return Ok(()); + } + let remaining = match self { + ParquetDecoderState::ReadingRowGroup { + remaining_row_groups, + } => remaining_row_groups, + ParquetDecoderState::DecodingRowGroup { .. } => { + return Err(ParquetError::General( + "swap_strategy called while a row group is being decoded; \ + check can_swap_strategy() first" + .to_string(), + )); + } + ParquetDecoderState::Finished => { + return Err(ParquetError::General( + "swap_strategy called on a finished decoder".to_string(), + )); + } + }; + if !remaining.is_at_row_group_boundary() { + return Err(ParquetError::General( + "swap_strategy called mid-row-group; check can_swap_strategy() first".to_string(), + )); + } + let StrategySwap { + projection, + filter, + row_selection_policy, + } = swap; + if let Some(projection) = projection { + remaining.set_projection(projection)?; + } + if let Some(filter) = filter { + remaining.set_filter(filter)?; + } + if let Some(policy) = row_selection_policy { + remaining.set_row_selection_policy(policy)?; + } + Ok(()) + } } #[cfg(test)] @@ -1476,6 +1643,167 @@ mod test { expect_finished(decoder.try_decode()); } + /// `swap_strategy` between row groups installs a new filter for the + /// next row group while leaving the just-decoded row group's results + /// untouched. + /// + /// Adaptive callers should drive the decoder with `try_next_reader` + /// rather than `try_decode`: `try_next_reader` returns once per row + /// group, giving the caller a clean window between two consecutive + /// returns to inspect stats and call `swap_strategy`. `try_decode` + /// barrels through row-group boundaries and is unsuitable for in-flight + /// strategy changes. + #[test] + fn test_swap_strategy_installs_filter_between_row_groups() { + let metadata = test_file_parquet_metadata(); + let schema_descr = metadata.file_metadata().schema_descr_ptr(); + + let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(metadata) + .unwrap() + .with_batch_size(1024) + .build() + .unwrap(); + + // Prefetch the entire file so we don't have to interleave I/O with + // the swap. PushBuffers carries the bytes through the swap. + decoder + .push_range(test_file_range(), TEST_FILE_DATA.clone()) + .unwrap(); + + // Reader for row group 0 — no filter. + let reader0 = expect_data(decoder.try_next_reader()); + let batches0: Vec<_> = reader0.collect::>().unwrap(); + let batch0 = concat_batches(&TEST_BATCH.schema(), &batches0).unwrap(); + assert_eq!(batch0.num_rows(), 200); + assert_eq!(batch0, TEST_BATCH.slice(0, 200)); + + // We're between row groups now. Swap in a filter on column "a". + assert!(decoder.can_swap_strategy()); + let filter = + ArrowPredicateFn::new(ProjectionMask::columns(&schema_descr, ["a"]), |batch| { + gt(batch.column(0), &Int64Array::new_scalar(250)) + }); + decoder + .swap_strategy( + StrategySwap::new().with_filter(Some(RowFilter::new(vec![Box::new(filter)]))), + ) + .unwrap(); + + // Reader for row group 1 — filter applied. Column "a" in RG1 has + // values 200..399; `a > 250` keeps 251..399 = 149 rows. + let reader1 = expect_data(decoder.try_next_reader()); + let batches1: Vec<_> = reader1.collect::>().unwrap(); + let batch1 = concat_batches(&TEST_BATCH.schema(), &batches1).unwrap(); + let expected1 = TEST_BATCH.slice(251, 149); + assert_eq!(batch1, expected1); + expect_finished(decoder.try_next_reader()); + } + + /// `swap_strategy` is rejected while a row group's reader is being + /// drained, and accepted again the moment we cross the boundary. + #[test] + fn test_swap_strategy_rejected_mid_row_group() { + let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()) + .unwrap() + .with_batch_size(50) + .build() + .unwrap(); + + decoder + .push_range(test_file_range(), TEST_FILE_DATA.clone()) + .unwrap(); + + // After getting the first batch, we're inside `DecodingRowGroup`: + // an active reader is still alive. Mid-reader is not a swap point. + let _ = expect_data(decoder.try_decode()); + assert!(!decoder.can_swap_strategy()); + + // Trying to swap mid-row-group is an error. + let err = decoder + .swap_strategy( + StrategySwap::new().with_row_selection_policy(super::RowSelectionPolicy::Mask), + ) + .unwrap_err(); + let err_msg = format!("{err}"); + assert!( + err_msg.contains("can_swap_strategy"), + "unexpected error: {err_msg}" + ); + + // Empty swap is a no-op even mid-row-group. + decoder.swap_strategy(StrategySwap::new()).unwrap(); + } + + /// `try_next_reader` hands the active reader off to the caller and + /// transitions the decoder back to `ReadingRowGroup` — so the caller + /// can call `swap_strategy` even while still iterating the returned + /// reader. (The handed-off reader has no link back to the decoder's + /// projection/filter; it has its own `ArrayReader` and `ReadPlan`.) + #[test] + fn test_swap_strategy_allowed_while_iterating_handed_off_reader() { + let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()) + .unwrap() + .with_batch_size(1024) + .build() + .unwrap(); + decoder + .push_range(test_file_range(), TEST_FILE_DATA.clone()) + .unwrap(); + + let reader0 = expect_data(decoder.try_next_reader()); + // Decoder no longer owns the reader, so it considers itself + // "between row groups" — swap is allowed even though the caller + // hasn't iterated `reader0` yet. + assert!(decoder.can_swap_strategy()); + // Iterating `reader0` is independent of the decoder's state. + let batches: Vec<_> = reader0.collect::>().unwrap(); + let batch0 = concat_batches(&TEST_BATCH.schema(), &batches).unwrap(); + assert_eq!(batch0, TEST_BATCH.slice(0, 200)); + } + + /// Bytes already buffered for a column survive a projection narrowing — + /// the next row group should not request any new data for the column + /// that was *already* in the projection. + #[test] + fn test_swap_strategy_preserves_buffered_bytes() { + let metadata = test_file_parquet_metadata(); + let schema_descr = metadata.file_metadata().schema_descr_ptr(); + + let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(metadata) + .unwrap() + .with_batch_size(1024) + .build() + .unwrap(); + + // Prefetch the whole file once. + decoder + .push_range(test_file_range(), TEST_FILE_DATA.clone()) + .unwrap(); + assert_eq!(decoder.buffered_bytes(), test_file_len()); + + // Drain RG0 with full projection via try_next_reader. + let reader0 = expect_data(decoder.try_next_reader()); + let _: Vec<_> = reader0.collect::>().unwrap(); + + // Narrow projection to just column "a" for RG1. + assert!(decoder.can_swap_strategy()); + decoder + .swap_strategy( + StrategySwap::new().with_projection(ProjectionMask::columns(&schema_descr, ["a"])), + ) + .unwrap(); + + // Column "a" bytes for RG1 were part of the original wide + // projection's prefetch; we should not need additional data. + let reader1 = expect_data(decoder.try_next_reader()); + let batches1: Vec<_> = reader1.collect::>().unwrap(); + // Schema is now just "a"; RG1 has values 200..399. + let expected1 = TEST_BATCH.slice(200, 200).project(&[0]).unwrap(); + let batch1 = concat_batches(&batches1[0].schema(), &batches1).unwrap(); + assert_eq!(batch1, expected1); + expect_finished(decoder.try_next_reader()); + } + /// Returns a batch with 400 rows, with 3 columns: "a", "b", "c" /// /// Note c is a different types (so the data page sizes will be different) diff --git a/parquet/src/arrow/push_decoder/reader_builder/mod.rs b/parquet/src/arrow/push_decoder/reader_builder/mod.rs index 0452cea436a3..d9d301161d61 100644 --- a/parquet/src/arrow/push_decoder/reader_builder/mod.rs +++ b/parquet/src/arrow/push_decoder/reader_builder/mod.rs @@ -296,6 +296,63 @@ impl RowGroupReaderBuilder { self.buffers.push_ranges(ranges, buffers); } + /// True iff the inner state is `Finished`. This is the only state in + /// which it is safe to swap projection / filter / row selection policy + /// because no `RowGroupInfo`, `FilterInfo`, or in-flight `DataRequest` + /// is referencing the previous values. + pub(crate) fn is_finished(&self) -> bool { + matches!(self.state, Some(RowGroupDecoderState::Finished)) + } + + /// Replace the projection mask used for subsequent row groups. + /// + /// Must only be called when [`Self::is_finished`]. `PushBuffers` are + /// preserved; bytes already fetched for columns that survive into the + /// new mask are reused. + pub(crate) fn set_projection( + &mut self, + projection: ProjectionMask, + ) -> Result<(), ParquetError> { + if !self.is_finished() { + return Err(ParquetError::General( + "RowGroupReaderBuilder::set_projection: state must be Finished".to_string(), + )); + } + self.projection = projection; + Ok(()) + } + + /// Replace the row filter used for subsequent row groups. + /// + /// Must only be called when [`Self::is_finished`]. Pass `None` to clear + /// the filter, `Some(filter)` to install a new one. + pub(crate) fn set_filter(&mut self, filter: Option) -> Result<(), ParquetError> { + if !self.is_finished() { + return Err(ParquetError::General( + "RowGroupReaderBuilder::set_filter: state must be Finished".to_string(), + )); + } + self.filter = filter; + Ok(()) + } + + /// Replace the row selection policy used for subsequent row groups. + /// + /// Must only be called when [`Self::is_finished`]. + pub(crate) fn set_row_selection_policy( + &mut self, + policy: RowSelectionPolicy, + ) -> Result<(), ParquetError> { + if !self.is_finished() { + return Err(ParquetError::General( + "RowGroupReaderBuilder::set_row_selection_policy: state must be Finished" + .to_string(), + )); + } + self.row_selection_policy = policy; + Ok(()) + } + /// Returns the total number of buffered bytes available pub fn buffered_bytes(&self) -> u64 { self.buffers.buffered_bytes() diff --git a/parquet/src/arrow/push_decoder/remaining.rs b/parquet/src/arrow/push_decoder/remaining.rs index 33e13abf9c12..aef105beb356 100644 --- a/parquet/src/arrow/push_decoder/remaining.rs +++ b/parquet/src/arrow/push_decoder/remaining.rs @@ -16,7 +16,10 @@ // under the License. use crate::DecodeResult; -use crate::arrow::arrow_reader::{ParquetRecordBatchReader, RowSelection}; +use crate::arrow::ProjectionMask; +use crate::arrow::arrow_reader::{ + ParquetRecordBatchReader, RowFilter, RowSelection, RowSelectionPolicy, +}; use crate::arrow::push_decoder::reader_builder::{ RowBudget, RowGroupBuildResult, RowGroupReaderBuilder, }; @@ -228,6 +231,40 @@ impl RemainingRowGroups { self.row_group_reader_builder.clear_all_ranges(); } + /// True iff the inner row-group reader is between row groups (state + /// `Finished`). Forward to [`RowGroupReaderBuilder::is_finished`]. + pub fn is_at_row_group_boundary(&self) -> bool { + self.row_group_reader_builder.is_finished() + } + + /// Number of row groups remaining (not including the one currently + /// being decoded). + pub fn row_groups_remaining(&self) -> usize { + self.frontier.row_groups.len() + } + + /// Replace the projection. Must be called when + /// [`Self::is_at_row_group_boundary`]. + pub fn set_projection(&mut self, projection: ProjectionMask) -> Result<(), ParquetError> { + self.row_group_reader_builder.set_projection(projection) + } + + /// Replace the row filter. Must be called when + /// [`Self::is_at_row_group_boundary`]. + pub fn set_filter(&mut self, filter: Option) -> Result<(), ParquetError> { + self.row_group_reader_builder.set_filter(filter) + } + + /// Replace the row selection policy. Must be called when + /// [`Self::is_at_row_group_boundary`]. + pub fn set_row_selection_policy( + &mut self, + policy: RowSelectionPolicy, + ) -> Result<(), ParquetError> { + self.row_group_reader_builder + .set_row_selection_policy(policy) + } + /// returns [`ParquetRecordBatchReader`] suitable for reading the next /// group of rows from the Parquet data, or the list of data ranges still /// needed to proceed From fee47441d266a86e839ce34ff4a8b4f00fcac0c1 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Mon, 27 Apr 2026 23:20:46 -0500 Subject: [PATCH 32/33] fix(parquet): drop private intra-doc link from swap_strategy docs `PushBuffers` is `pub(crate)` in the arrow-rs workspace, so the intra-doc link `[\`PushBuffers\`]: crate::util::push_buffers::PushBuffers` in `ParquetPushDecoder::swap_strategy`'s public docs failed the `-D rustdoc::private-intra-doc-links` check on `cargo +nightly doc --document-private-items`. The rest of the doc is unchanged; the prose now refers to the type by name without a link. Co-Authored-By: Claude Opus 4.7 (1M context) --- parquet/src/arrow/push_decoder/mod.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/parquet/src/arrow/push_decoder/mod.rs b/parquet/src/arrow/push_decoder/mod.rs index e62cf30e9aee..fe105f68cbde 100644 --- a/parquet/src/arrow/push_decoder/mod.rs +++ b/parquet/src/arrow/push_decoder/mod.rs @@ -406,16 +406,14 @@ impl ParquetPushDecoder { /// Returns `Err(ParquetError::General)` when called outside a row-group /// boundary; check [`Self::can_swap_strategy`] first. /// - /// The decoder's [`PushBuffers`] are preserved across the swap. Bytes - /// that have already been fetched for columns the new strategy still - /// needs will not be re-requested. Bytes for columns the new strategy - /// no longer needs remain buffered until [`Self::clear_all_ranges`] is - /// called or the decoder is dropped. + /// The decoder's internal `PushBuffers` are preserved across the swap. + /// Bytes that have already been fetched for columns the new strategy + /// still needs will not be re-requested. Bytes for columns the new + /// strategy no longer needs remain buffered until + /// [`Self::clear_all_ranges`] is called or the decoder is dropped. /// /// Limit, offset, batch size, metadata, fields, and predicate-cache size /// are unchanged by a swap. - /// - /// [`PushBuffers`]: crate::util::push_buffers::PushBuffers pub fn swap_strategy(&mut self, swap: StrategySwap) -> Result<(), ParquetError> { self.state.swap_strategy(swap) } From 752de6e86d1e1dfc9e0aec484647ea028337e43f Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 13 May 2026 01:31:30 -0400 Subject: [PATCH 33/33] test(parquet): cover projection swap with incremental I/O Add three swap_strategy tests: - expands_projection_between_row_groups: narrow -> wide swap with the whole file prefetched; verifies RG1 decodes the widened projection using already-buffered bytes. - expand_projection_requests_new_bytes: narrow -> wide swap driven incrementally; pins the post-swap NeedsData to the three RG1 column chunks (11168 bytes total), confirming the decoder fetches bytes for the newly-added columns. - narrow_projection_skips_unneeded_bytes: wide -> narrow swap driven incrementally; pins the post-swap NeedsData to the single RG1 column-a chunk (1856 bytes), confirming the decoder skips bytes for columns dropped from the projection. The incremental tests compare RG1-with-projection-X against RG1-with-projection-Y (not RG0 vs RG1), so the asymmetry isolates the projection change rather than per-row-group size variation in the StringView column "c". Expected ranges are hardcoded since TEST_BATCH and the writer settings are static. Co-Authored-By: Claude Opus 4.7 (1M context) --- parquet/src/arrow/push_decoder/mod.rs | 159 ++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) diff --git a/parquet/src/arrow/push_decoder/mod.rs b/parquet/src/arrow/push_decoder/mod.rs index fe105f68cbde..0fe67195a65d 100644 --- a/parquet/src/arrow/push_decoder/mod.rs +++ b/parquet/src/arrow/push_decoder/mod.rs @@ -1802,6 +1802,165 @@ mod test { expect_finished(decoder.try_next_reader()); } + /// Mirror of [`Self::test_swap_strategy_preserves_buffered_bytes`] in the + /// opposite direction: start with a narrow projection, drain RG0, then + /// widen the projection to add columns "b" and "c". Because the entire + /// file was prefetched, bytes for the newly-added columns are already + /// buffered and the next row group decodes without another `NeedsData`. + #[test] + fn test_swap_strategy_expands_projection_between_row_groups() { + let metadata = test_file_parquet_metadata(); + let schema_descr = metadata.file_metadata().schema_descr_ptr(); + + let builder = ParquetPushDecoderBuilder::try_new_decoder(metadata) + .unwrap() + .with_batch_size(1024); + let mut decoder = builder + .with_projection(ProjectionMask::columns(&schema_descr, ["a"])) + .build() + .unwrap(); + + // Prefetch the whole file up front. + decoder + .push_range(test_file_range(), TEST_FILE_DATA.clone()) + .unwrap(); + + // RG0 with narrow projection — just column "a". + let reader0 = expect_data(decoder.try_next_reader()); + let batches0: Vec<_> = reader0.collect::>().unwrap(); + let batch0 = concat_batches(&batches0[0].schema(), &batches0).unwrap(); + let expected0 = TEST_BATCH.slice(0, 200).project(&[0]).unwrap(); + assert_eq!(batch0, expected0); + + // At the boundary, widen to all three columns. + assert!(decoder.can_swap_strategy()); + decoder + .swap_strategy(StrategySwap::new().with_projection(ProjectionMask::columns( + &schema_descr, + ["a", "b", "c"], + ))) + .unwrap(); + + // Bytes for "b" and "c" in RG1 were part of the prefetch, so no + // further data should be requested. + let reader1 = expect_data(decoder.try_next_reader()); + let batches1: Vec<_> = reader1.collect::>().unwrap(); + let batch1 = concat_batches(&TEST_BATCH.schema(), &batches1).unwrap(); + assert_eq!(batch1, TEST_BATCH.slice(200, 200)); + expect_finished(decoder.try_next_reader()); + } + + /// Drive the decoder incrementally (no prefetch). Start with a narrow + /// projection, drain RG0 by satisfying its `NeedsData`, then widen the + /// projection to add columns "b" and "c". The post-swap `NeedsData` + /// for RG1 must request bytes for *all three* columns, not just the + /// originally-projected "a". The expected ranges are hardcoded + /// because `TEST_BATCH` and the writer settings are static; this + /// pins the layout cleanly without a parallel reference decoder. + #[test] + fn test_swap_strategy_expand_projection_requests_new_bytes() { + let metadata = test_file_parquet_metadata(); + let schema_descr = metadata.file_metadata().schema_descr_ptr(); + + let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(metadata) + .unwrap() + .with_batch_size(1024) + .with_projection(ProjectionMask::columns(&schema_descr, ["a"])) + .build() + .unwrap(); + + // RG0: incrementally satisfy the narrow request. + let ranges_rg0 = expect_needs_data(decoder.try_next_reader()); + // For a narrow ("a"-only) projection, RG0 is a single contiguous range. + assert_eq!(ranges_rg0, vec![4..1860]); + push_ranges_to_decoder(&mut decoder, ranges_rg0); + + let reader0 = expect_data(decoder.try_next_reader()); + let batches0: Vec<_> = reader0.collect::>().unwrap(); + let batch0 = concat_batches(&batches0[0].schema(), &batches0).unwrap(); + let expected0 = TEST_BATCH.slice(0, 200).project(&[0]).unwrap(); + assert_eq!(batch0, expected0); + // RG0's "a" pages have been consumed; no surviving buffer. + assert_eq!(decoder.buffered_bytes(), 0); + + // Widen the projection at the boundary. + assert!(decoder.can_swap_strategy()); + decoder + .swap_strategy(StrategySwap::new().with_projection(ProjectionMask::columns( + &schema_descr, + ["a", "b", "c"], + ))) + .unwrap(); + + // Post-swap RG1 request covers RG1's "a", "b", and "c" column + // chunks: ~1.8KiB each for "a" and "b", ~7.5KiB for the + // StringView column "c". Total 11168 bytes — far more than the + // 1856 bytes the narrow projection asked for in RG0. + let ranges_rg1 = expect_needs_data(decoder.try_next_reader()); + assert_eq!( + ranges_rg1, + vec![11062..12918, 12918..14774, 14774..22230] + ); + push_ranges_to_decoder(&mut decoder, ranges_rg1); + + let reader1 = expect_data(decoder.try_next_reader()); + let batches1: Vec<_> = reader1.collect::>().unwrap(); + let batch1 = concat_batches(&TEST_BATCH.schema(), &batches1).unwrap(); + assert_eq!(batch1, TEST_BATCH.slice(200, 200)); + expect_finished(decoder.try_next_reader()); + } + + /// Drive the decoder incrementally (no prefetch). Start with the full + /// projection, drain RG0 by satisfying its `NeedsData`, then narrow + /// the projection to just column "a". The post-swap `NeedsData` for + /// RG1 must request bytes for *only* column "a" — the same single + /// range a narrow projection would have asked for, not the three + /// ranges a wide projection would. + #[test] + fn test_swap_strategy_narrow_projection_skips_unneeded_bytes() { + let metadata = test_file_parquet_metadata(); + let schema_descr = metadata.file_metadata().schema_descr_ptr(); + + let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(metadata) + .unwrap() + .with_batch_size(1024) + .build() + .unwrap(); + + // RG0 with default (full) projection — three columns; three ranges. + let ranges_rg0 = expect_needs_data(decoder.try_next_reader()); + assert_eq!(ranges_rg0, vec![4..1860, 1860..3716, 3716..11062]); + push_ranges_to_decoder(&mut decoder, ranges_rg0); + + let reader0 = expect_data(decoder.try_next_reader()); + let batches0: Vec<_> = reader0.collect::>().unwrap(); + let batch0 = concat_batches(&TEST_BATCH.schema(), &batches0).unwrap(); + assert_eq!(batch0, TEST_BATCH.slice(0, 200)); + assert_eq!(decoder.buffered_bytes(), 0); + + // Narrow the projection at the boundary. + assert!(decoder.can_swap_strategy()); + decoder + .swap_strategy( + StrategySwap::new().with_projection(ProjectionMask::columns(&schema_descr, ["a"])), + ) + .unwrap(); + + // Post-swap RG1 request asks for column "a" only — a single + // 1856-byte range, not the three column-chunk ranges the wide + // projection would have produced. + let ranges_rg1 = expect_needs_data(decoder.try_next_reader()); + assert_eq!(ranges_rg1, vec![11062..12918]); + push_ranges_to_decoder(&mut decoder, ranges_rg1); + + let reader1 = expect_data(decoder.try_next_reader()); + let batches1: Vec<_> = reader1.collect::>().unwrap(); + let batch1 = concat_batches(&batches1[0].schema(), &batches1).unwrap(); + let expected1 = TEST_BATCH.slice(200, 200).project(&[0]).unwrap(); + assert_eq!(batch1, expected1); + expect_finished(decoder.try_next_reader()); + } + /// Returns a batch with 400 rows, with 3 columns: "a", "b", "c" /// /// Note c is a different types (so the data page sizes will be different)