From 4eb724ee27acd3ec949f863b3930a8f744bfd48a Mon Sep 17 00:00:00 2001 From: Vegard Stikbakke Date: Wed, 18 Feb 2026 21:43:00 +0100 Subject: [PATCH 1/2] Implement native interleave for ListView --- arrow-select/src/interleave.rs | 107 +++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/arrow-select/src/interleave.rs b/arrow-select/src/interleave.rs index 1bd34a7f3a58..b8e15861cb65 100644 --- a/arrow-select/src/interleave.rs +++ b/arrow-select/src/interleave.rs @@ -114,6 +114,8 @@ pub fn interleave( DataType::Int64 => interleave_run_end::(values, indices), t => unreachable!("illegal run-end type {t}"), }, + DataType::ListView(field) => interleave_list_view::(values, indices, field), + DataType::LargeListView(field) => interleave_list_view::(values, indices, field), _ => interleave_fallback(values, indices) } } @@ -482,6 +484,56 @@ fn interleave_run_end( )?)) } +fn interleave_list_view( + values: &[&dyn Array], + indices: &[(usize, usize)], + field: &FieldRef, +) -> Result { + let interleaved = Interleave::<'_, GenericListViewArray>::new(values, indices); + + // Build new offsets/sizes and compute total child capacity + let mut capacity = 0usize; + let mut offsets = Vec::with_capacity(indices.len()); + let mut sizes = Vec::with_capacity(indices.len()); + for &(array_idx, row_idx) in indices { + let list = interleaved.arrays[array_idx]; + let size = list.sizes()[row_idx].as_usize(); + offsets.push( + O::from_usize(capacity).ok_or_else(|| ArrowError::OffsetOverflowError(capacity))?, + ); + sizes.push(O::from_usize(size).ok_or_else(|| ArrowError::OffsetOverflowError(size))?); + capacity += size; + } + + // Bulk-copy child value ranges into a single flat child array + let child_data: Vec<_> = interleaved + .arrays + .iter() + .map(|list| list.values().to_data()) + .collect(); + let child_data_refs: Vec<_> = child_data.iter().collect(); + let mut mutable_child = MutableArrayData::new(child_data_refs, false, capacity); + for &(array_idx, row_idx) in indices { + let list = interleaved.arrays[array_idx]; + let start = list.offsets()[row_idx].as_usize(); + let size = list.sizes()[row_idx].as_usize(); + if size > 0 { + mutable_child.extend(array_idx, start, start + size); + } + } + + let interleaved_values = make_array(mutable_child.freeze()); + let list_view_array = GenericListViewArray::::new( + field.clone(), + offsets.into(), + sizes.into(), + interleaved_values, + interleaved.nulls, + ); + + Ok(Arc::new(list_view_array)) +} + /// Fallback implementation of interleave using [`MutableArrayData`] fn interleave_fallback( values: &[&dyn Array], @@ -841,6 +893,61 @@ mod tests { test_interleave_lists::(); } + fn test_interleave_list_views() { + // [[1, 2], null, [3]] + let mut a = GenericListBuilder::::new(Int32Builder::new()); + a.values().append_value(1); + a.values().append_value(2); + a.append(true); + a.append(false); + a.values().append_value(3); + a.append(true); + let a: GenericListViewArray = a.finish().into(); + + // [[4], null, [5, 6, null]] + let mut b = GenericListBuilder::::new(Int32Builder::new()); + b.values().append_value(4); + b.append(true); + b.append(false); + b.values().append_value(5); + b.values().append_value(6); + b.values().append_null(); + b.append(true); + let b: GenericListViewArray = b.finish().into(); + + let values = interleave(&[&a, &b], &[(0, 2), (0, 1), (1, 0), (1, 2), (1, 1)]).unwrap(); + let v = values + .as_any() + .downcast_ref::>() + .unwrap(); + + // [[3], null, [4], [5, 6, null], null] + let mut expected = GenericListBuilder::::new(Int32Builder::new()); + expected.values().append_value(3); + expected.append(true); + expected.append(false); + expected.values().append_value(4); + expected.append(true); + expected.values().append_value(5); + expected.values().append_value(6); + expected.values().append_null(); + expected.append(true); + expected.append(false); + let expected: GenericListViewArray = expected.finish().into(); + + assert_eq!(v, &expected); + } + + #[test] + fn test_list_views() { + test_interleave_list_views::(); + } + + #[test] + fn test_large_list_views() { + test_interleave_list_views::(); + } + #[test] fn test_struct_without_nulls() { let fields = Fields::from(vec![ From 5d539d3f1d03e487e12614367aea4bedc40c56f0 Mon Sep 17 00:00:00 2001 From: Alfonso Subiotto Marques Date: Wed, 29 Apr 2026 12:30:31 +0200 Subject: [PATCH 2/2] arrow-select: preserve ListView element sharing in interleave The previous implementation copies child elements per-row via MutableArrayData::extend(), destroying overlapping offset/size sharing. This matters for merge sorts over data like stacktrace profiles where many rows reference the same backing elements. Use a hybrid strategy: compare concat cost (sum of source backing array lengths) vs per-row cost (sum of selected row sizes). When sharing exists (per-row cost > concat cost), concatenate backing arrays and adjust offsets to preserve sharing. Otherwise use the per-row copy. Adds overlapping ListView test. Signed-off-by: Alfonso Subiotto Marques --- arrow-select/src/interleave.rs | 141 +++++++++++++++++++++++++++++++-- 1 file changed, 133 insertions(+), 8 deletions(-) diff --git a/arrow-select/src/interleave.rs b/arrow-select/src/interleave.rs index b8e15861cb65..13d28747dfcf 100644 --- a/arrow-select/src/interleave.rs +++ b/arrow-select/src/interleave.rs @@ -491,7 +491,29 @@ fn interleave_list_view( ) -> Result { let interleaved = Interleave::<'_, GenericListViewArray>::new(values, indices); - // Build new offsets/sizes and compute total child capacity + // Pick whichever strategy produces fewer child elements: + // - Per-row copy: total = sum of selected sizes. Better for sparse selections. + // - Concat + offset adjustment: total = sum of source backing array lengths. + // Better when rows share backing elements via overlapping offset/size ranges. + let concat_cost: usize = interleaved.arrays.iter().map(|lv| lv.values().len()).sum(); + let per_row_cost: usize = indices + .iter() + .map(|&(a, r)| interleaved.arrays[a].sizes()[r].as_usize()) + .sum(); + + if per_row_cost <= concat_cost { + interleave_list_view_copy::(&interleaved, indices, field) + } else { + interleave_list_view_concat::(&interleaved, indices, field) + } +} + +/// Per-row copy: copies each selected row's child elements into a new flat array. +fn interleave_list_view_copy( + interleaved: &Interleave<'_, GenericListViewArray>, + indices: &[(usize, usize)], + field: &FieldRef, +) -> Result { let mut capacity = 0usize; let mut offsets = Vec::with_capacity(indices.len()); let mut sizes = Vec::with_capacity(indices.len()); @@ -505,7 +527,6 @@ fn interleave_list_view( capacity += size; } - // Bulk-copy child value ranges into a single flat child array let child_data: Vec<_> = interleaved .arrays .iter() @@ -522,16 +543,53 @@ fn interleave_list_view( } } - let interleaved_values = make_array(mutable_child.freeze()); - let list_view_array = GenericListViewArray::::new( + Ok(Arc::new(GenericListViewArray::::new( field.clone(), offsets.into(), sizes.into(), - interleaved_values, - interleaved.nulls, - ); + make_array(mutable_child.freeze()), + interleaved.nulls.clone(), + ))) +} + +/// Concat backing arrays: concatenates all source value arrays and adjusts offsets. +/// Preserves within-source element sharing. +fn interleave_list_view_concat( + interleaved: &Interleave<'_, GenericListViewArray>, + indices: &[(usize, usize)], + field: &FieldRef, +) -> Result { + let child_arrays: Vec<&dyn Array> = interleaved + .arrays + .iter() + .map(|lv| lv.values().as_ref()) + .collect(); + let mut base_offsets = Vec::with_capacity(interleaved.arrays.len()); + let mut running = 0usize; + for lv in &interleaved.arrays { + base_offsets.push(running); + running += lv.values().len(); + } + let combined_values = concat(&child_arrays)?; - Ok(Arc::new(list_view_array)) + let mut new_offsets = Vec::with_capacity(indices.len()); + let mut new_sizes = Vec::with_capacity(indices.len()); + for &(array_idx, row_idx) in indices { + let lv = interleaved.arrays[array_idx]; + let adjusted = lv.offsets()[row_idx].as_usize() + base_offsets[array_idx]; + new_offsets.push( + O::from_usize(adjusted).ok_or_else(|| ArrowError::OffsetOverflowError(adjusted))?, + ); + new_sizes.push(lv.sizes()[row_idx]); + } + + Ok(Arc::new(GenericListViewArray::::new( + field.clone(), + new_offsets.into(), + new_sizes.into(), + combined_values, + interleaved.nulls.clone(), + ))) } /// Fallback implementation of interleave using [`MutableArrayData`] @@ -948,6 +1006,73 @@ mod tests { test_interleave_list_views::(); } + #[test] + fn test_interleave_list_view_overlapping() { + let field = Arc::new(Field::new_list_field(DataType::Int64, false)); + + // lv_a: 10 rows, two groups of 5 sharing the same backing elements. + // rows 0-4 → offset 0, size 5 → [0,1,2,3,4] + // rows 5-9 → offset 5, size 5 → [5,6,7,8,9] + let lv_a = ListViewArray::new( + Arc::clone(&field), + ScalarBuffer::from(vec![0i32, 0, 0, 0, 0, 5, 5, 5, 5, 5]), + ScalarBuffer::from(vec![5i32; 10]), + Arc::new(Int64Array::from_iter_values(0..10)), + None, + ); + + // lv_b: 8 rows, two groups of 4 sharing the same backing elements. + // rows 0-3 → offset 0, size 3 → [100,101,102] + // rows 4-7 → offset 3, size 3 → [103,104,105] + let lv_b = ListViewArray::new( + Arc::clone(&field), + ScalarBuffer::from(vec![0i32, 0, 0, 0, 3, 3, 3, 3]), + ScalarBuffer::from(vec![3i32; 8]), + Arc::new(Int64Array::from_iter_values(100..106)), + None, + ); + + let indices: Vec<(usize, usize)> = vec![ + (0, 0), + (1, 0), + (0, 5), + (1, 4), + (0, 1), + (1, 1), + (0, 6), + (1, 5), + ]; + let result = interleave(&[&lv_a as &dyn Array, &lv_b as &dyn Array], &indices).unwrap(); + result + .to_data() + .validate_full() + .expect("result must be valid"); + + let result_lv = result.as_list_view::(); + assert_eq!(result_lv.len(), 8); + assert_eq!( + result_lv.value(0).as_primitive::().values(), + &[0, 1, 2, 3, 4] + ); + assert_eq!( + result_lv.value(1).as_primitive::().values(), + &[100, 101, 102] + ); + assert_eq!( + result_lv.value(2).as_primitive::().values(), + &[5, 6, 7, 8, 9] + ); + assert_eq!( + result_lv.value(3).as_primitive::().values(), + &[103, 104, 105] + ); + + // Backing elements = sum of source arrays (10 + 6 = 16), not per-row + // expansion (8 rows × avg ~4 = 32). Overlapping sharing is preserved. + let total_input_elements = lv_a.values().len() + lv_b.values().len(); + assert_eq!(result_lv.values().len(), total_input_elements); + } + #[test] fn test_struct_without_nulls() { let fields = Fields::from(vec![