From 6cadf3b4de916c707e2103b123a168154e668a33 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 20 Mar 2026 15:00:39 -0400 Subject: [PATCH 01/17] Prepare for 58.1.0 Release (#9573) # Which issue does this PR close? - part of https://github.com/apache/arrow-rs/issues/9108 # Rationale for this change Prepare for next release # What changes are included in this PR? 1. Update version to `58.1.0` 2. Add changelog. See rendered preview here: https://github.com/alamb/arrow-rs/blob/alamb/prepare_58.1.0/CHANGELOG.md # Are these changes tested? By CI # Are there any user-facing changes? Yes --- CHANGELOG-old.md | 196 ++++++++++++++++++++++ CHANGELOG.md | 273 +++++++++++-------------------- Cargo.toml | 42 ++--- dev/release/update_change_log.sh | 4 +- 4 files changed, 317 insertions(+), 198 deletions(-) diff --git a/CHANGELOG-old.md b/CHANGELOG-old.md index 300c1f4b2e40..273884ea1fa6 100644 --- a/CHANGELOG-old.md +++ b/CHANGELOG-old.md @@ -19,6 +19,202 @@ # Historical Changelog + +## [58.0.0](https://github.com/apache/arrow-rs/tree/58.0.0) (2026-02-19) + +[Full Changelog](https://github.com/apache/arrow-rs/compare/57.3.0...58.0.0) + +**Breaking changes:** + +- Remove support for List types in bit\_length kernel [\#9350](https://github.com/apache/arrow-rs/pull/9350) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([codephage2020](https://github.com/codephage2020)) +- Optimize `from_bitwise_unary_op` [\#9297](https://github.com/apache/arrow-rs/pull/9297) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan)) +- Mark `BufferBuilder::new_from_buffer` as unsafe [\#9292](https://github.com/apache/arrow-rs/pull/9292) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Jefffrey](https://github.com/Jefffrey)) +- \[Variant\] Support `['fieldName']` in VariantPath parser [\#9276](https://github.com/apache/arrow-rs/pull/9276) ([klion26](https://github.com/klion26)) +- Remove parquet arrow\_cast dependency [\#9077](https://github.com/apache/arrow-rs/pull/9077) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([tustvold](https://github.com/tustvold)) +- feat: change default behavior for Parquet `PageEncodingStats` to bitmask [\#9051](https://github.com/apache/arrow-rs/pull/9051) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([WaterWhisperer](https://github.com/WaterWhisperer)) +- \[arrow\] Minimize allocation in GenericViewArray::slice\(\) [\#9016](https://github.com/apache/arrow-rs/pull/9016) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([maxburke](https://github.com/maxburke)) + +**Implemented enhancements:** + +- Avoid allocating a `Vec` in `StructBuilder` [\#9427](https://github.com/apache/arrow-rs/issues/9427) +- Zstd context reuse [\#9401](https://github.com/apache/arrow-rs/issues/9401) +- Optimize `from_bitwise_unary_op` [\#9364](https://github.com/apache/arrow-rs/issues/9364) +- Support `RunEndEncoded` in ord comparator [\#9360](https://github.com/apache/arrow-rs/issues/9360) +- Support `RunEndEncoded` arrays in `arrow-json` [\#9359](https://github.com/apache/arrow-rs/issues/9359) +- Support `BinaryView` in `bit_length` kernel [\#9351](https://github.com/apache/arrow-rs/issues/9351) +- Remove support for `List` types in `bit_length` kernel [\#9349](https://github.com/apache/arrow-rs/issues/9349) +- Support roundtrip `ListView` in parquet arrow writer [\#9344](https://github.com/apache/arrow-rs/issues/9344) +- Support `ListView` in `length` kernel [\#9343](https://github.com/apache/arrow-rs/issues/9343) +- Support `ListView` in sort kernel [\#9341](https://github.com/apache/arrow-rs/issues/9341) +- Add some way to create a Timestamp from a `DateTime` [\#9337](https://github.com/apache/arrow-rs/issues/9337) +- Introduce `DataType::is_list` and `DataType::IsBinary` [\#9326](https://github.com/apache/arrow-rs/issues/9326) +- Performance of creating all null dictionary array can be improved [\#9321](https://github.com/apache/arrow-rs/issues/9321) +- \[arrow-avro\] Add missing Arrow DataType support with `avro_custom_types` round-trip + non-custom fallbacks [\#9290](https://github.com/apache/arrow-rs/issues/9290) + +**Fixed bugs:** + +- ArrowArrayStreamReader errors on zero-column record batches [\#9394](https://github.com/apache/arrow-rs/issues/9394) +- Regression on main \(58\): Parquet argument error: Parquet error: Required field type\_ is missing [\#9315](https://github.com/apache/arrow-rs/issues/9315) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] + +**Documentation updates:** + +- Improve safety documentation of the `Array` trait [\#9314](https://github.com/apache/arrow-rs/pull/9314) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Improve docs and add build\(\) method to `{Null,Boolean,}BufferBuilder` [\#9155](https://github.com/apache/arrow-rs/pull/9155) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Improve `ArrowReaderBuilder::with_row_filter` documentation [\#9153](https://github.com/apache/arrow-rs/pull/9153) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([alamb](https://github.com/alamb)) +- docs: Improve main README.md and highlight community [\#9119](https://github.com/apache/arrow-rs/pull/9119) ([alamb](https://github.com/alamb)) +- Docs: Add additional documentation and example for `make_array` [\#9112](https://github.com/apache/arrow-rs/pull/9112) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- doc: fix link on FixedSizeListArray doc [\#9033](https://github.com/apache/arrow-rs/pull/9033) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Jefffrey](https://github.com/Jefffrey)) + +**Performance improvements:** + +- Replace `ArrayData` with direct Array construction [\#9338](https://github.com/apache/arrow-rs/pull/9338) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- Remove some `unsafe` and allocations when creating PrimitiveArrays from Vec and `from_trusted_len_iter` [\#9299](https://github.com/apache/arrow-rs/pull/9299) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- parquet: rle skip decode loop when batch contains all max levels \(aka no nulls\) [\#9258](https://github.com/apache/arrow-rs/pull/9258) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([lyang24](https://github.com/lyang24)) +- Improve parquet BinaryView / StringView decoder performance \(up to -35%\) [\#9236](https://github.com/apache/arrow-rs/pull/9236) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) +- Avoid a clone when creating `BooleanArray` from ArrayData [\#9159](https://github.com/apache/arrow-rs/pull/9159) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Avoid overallocating arrays in coalesce primitives / views [\#9132](https://github.com/apache/arrow-rs/pull/9132) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan)) +- perf: Avoid ArrayData allocation in PrimitiveArray::reinterpret\_cast [\#9129](https://github.com/apache/arrow-rs/pull/9129) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- \[Parquet\] perf: Create StructArrays directly rather than via `ArrayData` \(1% improvement\) [\#9120](https://github.com/apache/arrow-rs/pull/9120) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Avoid clones in `make_array` for `StructArray` and `GenericByteViewArray` [\#9114](https://github.com/apache/arrow-rs/pull/9114) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- perf: optimize hex decoding in json \(1.8x faster in binary-heavy\) [\#9091](https://github.com/apache/arrow-rs/pull/9091) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Weijun-H](https://github.com/Weijun-H)) +- Speed up binary kernels \(30% faster `and` and `or`\), add `BooleanBuffer::from_bitwise_binary_op` [\#9090](https://github.com/apache/arrow-rs/pull/9090) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- perf: improve field indexing in JSON StructArrayDecoder \(1.7x speed up\) [\#9086](https://github.com/apache/arrow-rs/pull/9086) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Weijun-H](https://github.com/Weijun-H)) +- bench: added to row\_format benchmark conversion of 53 non-nested columns [\#9081](https://github.com/apache/arrow-rs/pull/9081) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) +- perf: improve calculating length performance for view byte array in row conversion [\#9080](https://github.com/apache/arrow-rs/pull/9080) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) +- perf: improve calculating length performance for nested arrays in row conversion [\#9079](https://github.com/apache/arrow-rs/pull/9079) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) +- perf: improve calculating length performance for `GenericByteArray` in row conversion [\#9078](https://github.com/apache/arrow-rs/pull/9078) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) + +**Closed issues:** + +- BatchCoalescer::push\_batch panics on schema mismatch instead of returning error [\#9389](https://github.com/apache/arrow-rs/issues/9389) +- Release arrow-rs / parquet Minor version `57.3.0` \(January 2026\) [\#9240](https://github.com/apache/arrow-rs/issues/9240) +- \[Variant\] support `..` and `['fieldName']` syntax in the VariantPath parser [\#9050](https://github.com/apache/arrow-rs/issues/9050) +- Support Float16 for create\_random\_array [\#9028](https://github.com/apache/arrow-rs/issues/9028) + +**Merged pull requests:** + +- Avoid allocating a `Vec` in `StructBuilder` [\#9428](https://github.com/apache/arrow-rs/pull/9428) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Fokko](https://github.com/Fokko)) +- fix: fixed trait functions clash get\_date\_time\_part\_extract\_fn \(\#8221\) [\#9424](https://github.com/apache/arrow-rs/pull/9424) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([esavier](https://github.com/esavier)) +- \[Minor\] Use per-predicate projection masks in arrow\_reader\_clickbench benchmark [\#9413](https://github.com/apache/arrow-rs/pull/9413) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) +- Fix `ArrowArrayStreamReader` for 0-columns record batch streams [\#9405](https://github.com/apache/arrow-rs/pull/9405) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([jonded94](https://github.com/jonded94)) +- Use zstd::bulk API in IPC and Parquet with context reuse for compression and decompression [\#9400](https://github.com/apache/arrow-rs/pull/9400) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan)) +- Reproduce the issue of \#9370 in a minimal, end-to-end way [\#9399](https://github.com/apache/arrow-rs/pull/9399) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([jonded94](https://github.com/jonded94)) +- perf: optimize skipper for varint values used when projecting Avro record types [\#9397](https://github.com/apache/arrow-rs/pull/9397) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mzabaluev](https://github.com/mzabaluev)) +- fix: return error instead of panic on schema mismatch in BatchCoalescer::push\_batch [\#9390](https://github.com/apache/arrow-rs/pull/9390) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([bvolpato-dd](https://github.com/bvolpato-dd)) +- Minor: Add additional test coverage for WriterProperties::{max\_row\_group\_row\_count,max\_row\_group\_size} [\#9387](https://github.com/apache/arrow-rs/pull/9387) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([alamb](https://github.com/alamb)) +- Moving invalid\_utf8 tests into a separate mod [\#9384](https://github.com/apache/arrow-rs/pull/9384) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([sdf-jkl](https://github.com/sdf-jkl)) +- Update sysinfo requirement from 0.37.1 to 0.38.1 [\#9383](https://github.com/apache/arrow-rs/pull/9383) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([dependabot[bot]](https://github.com/apps/dependabot)) +- feat: support RunEndEncoded arrays in arrow-json reader and writer [\#9379](https://github.com/apache/arrow-rs/pull/9379) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Abhisheklearn12](https://github.com/Abhisheklearn12)) +- Remove lint issues in parquet-related code. [\#9375](https://github.com/apache/arrow-rs/pull/9375) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([brunal](https://github.com/brunal)) +- Add RunEndEncoded array comparator [\#9368](https://github.com/apache/arrow-rs/pull/9368) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([codephage2020](https://github.com/codephage2020)) +- feat: support BinaryView in bit\_length kernel [\#9363](https://github.com/apache/arrow-rs/pull/9363) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Abhisheklearn12](https://github.com/Abhisheklearn12)) +- Add regression tests for Parquet large binary offset overflow [\#9361](https://github.com/apache/arrow-rs/pull/9361) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([vigneshsiva11](https://github.com/vigneshsiva11)) +- feat: add max\_row\_group\_bytes option to WriterProperties [\#9357](https://github.com/apache/arrow-rs/pull/9357) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([yonipeleg33](https://github.com/yonipeleg33)) +- doc: remove disclaimer about `ListView` not being fully supported [\#9356](https://github.com/apache/arrow-rs/pull/9356) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Jefffrey](https://github.com/Jefffrey)) +- Move row\_filter async tests from parquet async reader [\#9355](https://github.com/apache/arrow-rs/pull/9355) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([sdf-jkl](https://github.com/sdf-jkl)) +- \[Parquet\] Allow setting page size per column [\#9353](https://github.com/apache/arrow-rs/pull/9353) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([XiangpengHao](https://github.com/XiangpengHao)) +- feat: Support roundtrip ListView in parquet arrow writer [\#9352](https://github.com/apache/arrow-rs/pull/9352) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([codephage2020](https://github.com/codephage2020)) +- feat: add ListView and LargeListView support to arrow-ord [\#9347](https://github.com/apache/arrow-rs/pull/9347) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([codephage2020](https://github.com/codephage2020)) +- Support ListView in length kernel [\#9346](https://github.com/apache/arrow-rs/pull/9346) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([vegarsti](https://github.com/vegarsti)) +- feat: Add from\_datetime method to Timestamp types [\#9345](https://github.com/apache/arrow-rs/pull/9345) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([codephage2020](https://github.com/codephage2020)) +- \[main\] Update version to 57.3.0, add changelog [\#9334](https://github.com/apache/arrow-rs/pull/9334) ([alamb](https://github.com/alamb)) +- build\(deps\): update pyo3 requirement from 0.27.1 to 0.28.0 [\#9331](https://github.com/apache/arrow-rs/pull/9331) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Add `DataType::is_list` and `DataType::is_binary` [\#9327](https://github.com/apache/arrow-rs/pull/9327) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([AdamGS](https://github.com/AdamGS)) +- Fix string array equality when the values buffer is the same and only the offsets to access it differ [\#9325](https://github.com/apache/arrow-rs/pull/9325) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([jhorstmann](https://github.com/jhorstmann)) +- perf: skip validation of dictionary keys if all null [\#9322](https://github.com/apache/arrow-rs/pull/9322) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([albertlockett](https://github.com/albertlockett)) +- parquet: use rwlock instead of mutex in predicate cache [\#9319](https://github.com/apache/arrow-rs/pull/9319) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([lyang24](https://github.com/lyang24)) +- nit: remove usused code [\#9318](https://github.com/apache/arrow-rs/pull/9318) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([lyang24](https://github.com/lyang24)) +- Remove unnecessary Arc\ [\#9316](https://github.com/apache/arrow-rs/pull/9316) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([scovich](https://github.com/scovich)) +- Optimize data page statistics conversion \(up to 4x\) [\#9303](https://github.com/apache/arrow-rs/pull/9303) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan)) +- \[regression\] Error with adaptive predicate pushdown: "Invalid offset in sparse column chunk data: 754, no matching page found." [\#9301](https://github.com/apache/arrow-rs/pull/9301) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([sdf-jkl](https://github.com/sdf-jkl)) +- Improve `PrimitiveArray::from_iter` perf [\#9294](https://github.com/apache/arrow-rs/pull/9294) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan)) +- Add additional Arrow type support [\#9291](https://github.com/apache/arrow-rs/pull/9291) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([jecsand838](https://github.com/jecsand838)) +- fix: ensure `BufferBuilder::truncate` doesn't overset length [\#9288](https://github.com/apache/arrow-rs/pull/9288) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Jefffrey](https://github.com/Jefffrey)) +- Add benchmark for row group index reader perf [\#9285](https://github.com/apache/arrow-rs/pull/9285) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([friendlymatthew](https://github.com/friendlymatthew)) +- fix union array row converter to handle non-sequential type ids [\#9283](https://github.com/apache/arrow-rs/pull/9283) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([friendlymatthew](https://github.com/friendlymatthew)) +- parquet: reduce clone in delta byte array decoder [\#9282](https://github.com/apache/arrow-rs/pull/9282) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([lyang24](https://github.com/lyang24)) +- fix: fix \[\[NULL\]\] array doesn't roundtrip in arrow-row bug [\#9275](https://github.com/apache/arrow-rs/pull/9275) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([lichuang](https://github.com/lichuang)) +- Enhance list casting, adding more cases for list views [\#9274](https://github.com/apache/arrow-rs/pull/9274) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Jefffrey](https://github.com/Jefffrey)) +- \[Variant\] Add path index access tests for list [\#9273](https://github.com/apache/arrow-rs/pull/9273) ([liamzwbao](https://github.com/liamzwbao)) +- Factor out json reader's static make\_decoder args to a struct [\#9271](https://github.com/apache/arrow-rs/pull/9271) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([scovich](https://github.com/scovich)) +- make\_decoder accepts borrowed DataType instead of owned [\#9270](https://github.com/apache/arrow-rs/pull/9270) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([scovich](https://github.com/scovich)) +- Implement a more generic from\_nested\_iter method for list arrays [\#9268](https://github.com/apache/arrow-rs/pull/9268) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([jhorstmann](https://github.com/jhorstmann)) +- Move extension type construction logic out of Field [\#9266](https://github.com/apache/arrow-rs/pull/9266) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([scovich](https://github.com/scovich)) +- fix: support casting string to f16 [\#9262](https://github.com/apache/arrow-rs/pull/9262) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Jefffrey](https://github.com/Jefffrey)) +- Add additional coverage for StringViewArray comparisons [\#9257](https://github.com/apache/arrow-rs/pull/9257) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Upgrade to object store 0.13.1 [\#9256](https://github.com/apache/arrow-rs/pull/9256) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) +- \[Parquet\] test adaptive predicate pushdown with skipped page [\#9251](https://github.com/apache/arrow-rs/pull/9251) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([sdf-jkl](https://github.com/sdf-jkl)) +- Speed up string view comparison \(up to 3x\) [\#9250](https://github.com/apache/arrow-rs/pull/9250) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan)) +- Add bench for LocalFileSystem [\#9248](https://github.com/apache/arrow-rs/pull/9248) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) +- \[Parquet\] Add test for reading/writing long UTF8 StringViews [\#9246](https://github.com/apache/arrow-rs/pull/9246) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([alamb](https://github.com/alamb)) +- \[Parquet\] test adaptive predicate pushdown with skipped page [\#9243](https://github.com/apache/arrow-rs/pull/9243) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([erratic-pattern](https://github.com/erratic-pattern)) +- Add tests and fixes for schema resolution bug [\#9237](https://github.com/apache/arrow-rs/pull/9237) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([jecsand838](https://github.com/jecsand838)) +- Revert "Seal Array trait \(\#9092\)", mark `Array` as `unsafe` [\#9234](https://github.com/apache/arrow-rs/pull/9234) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([gabotechs](https://github.com/gabotechs)) +- Speedup filter \(up to ~1.5x\) `FilterBuilder::Optimize`/`BitIndexIterator`/`iter_set_bits_rev` [\#9229](https://github.com/apache/arrow-rs/pull/9229) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan)) +- optimize `RowGroupIndexReader` for single row group reads [\#9226](https://github.com/apache/arrow-rs/pull/9226) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([friendlymatthew](https://github.com/friendlymatthew)) +- test: improve arrow-row fuzz tests [\#9222](https://github.com/apache/arrow-rs/pull/9222) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) +- arrow-cast: support packing to Dictionary\(\_, Utf8View/BinaryView\) [\#9220](https://github.com/apache/arrow-rs/pull/9220) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([ethan-tyler](https://github.com/ethan-tyler)) +- Add additional test coverage for `BatchCoalescer` push\_batch\_with\_filter [\#9218](https://github.com/apache/arrow-rs/pull/9218) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- \[Parquet\] Optimize appending max level comparison in DefinitionLevelDecoder [\#9217](https://github.com/apache/arrow-rs/pull/9217) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([jhorstmann](https://github.com/jhorstmann)) +- Remove dead code to fix clippy failure on main [\#9215](https://github.com/apache/arrow-rs/pull/9215) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- \[Parquet\] perf: reuse seeked File clone in ChunkReader::get\_read\(\) [\#9214](https://github.com/apache/arrow-rs/pull/9214) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([fvaleye](https://github.com/fvaleye)) +- fix: \[9018\]Fixed RunArray slice offsets\(row, cast, eq\) [\#9213](https://github.com/apache/arrow-rs/pull/9213) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([manishkr](https://github.com/manishkr)) +- Add benchmarks for reading struct arrays from parquet [\#9210](https://github.com/apache/arrow-rs/pull/9210) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([jhorstmann](https://github.com/jhorstmann)) +- Support casting negative scale decimals to numeric [\#9207](https://github.com/apache/arrow-rs/pull/9207) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Chiicake](https://github.com/Chiicake)) +- Deprecate `ArrowReaderOptions::with_page_index` and update API [\#9199](https://github.com/apache/arrow-rs/pull/9199) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([friendlymatthew](https://github.com/friendlymatthew)) +- arrow-ipc: add reset method to DictionaryTracker [\#9196](https://github.com/apache/arrow-rs/pull/9196) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([albertlockett](https://github.com/albertlockett)) +- Avoid a clone when creating `ListArray` from ArrayData [\#9194](https://github.com/apache/arrow-rs/pull/9194) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Avoid a clone when creating `ListViewArray` from ArrayData [\#9193](https://github.com/apache/arrow-rs/pull/9193) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Avoid a clone when creating `MapArray` from ArrayData [\#9192](https://github.com/apache/arrow-rs/pull/9192) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Avoid a clone when creating `NullArray` from ArrayData [\#9191](https://github.com/apache/arrow-rs/pull/9191) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Avoid a clone when creating `PrimitiveArray` from ArrayData [\#9190](https://github.com/apache/arrow-rs/pull/9190) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Avoid a clone when creating `RunEndArray` from ArrayData [\#9189](https://github.com/apache/arrow-rs/pull/9189) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Avoid a clone when creating `UnionArray` from ArrayData [\#9188](https://github.com/apache/arrow-rs/pull/9188) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Avoid a clone when creating `FixedSizeListArray` from ArrayData [\#9187](https://github.com/apache/arrow-rs/pull/9187) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Avoid a clone when creating `FixedSizeBinaryArray` from ArrayData [\#9186](https://github.com/apache/arrow-rs/pull/9186) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Avoid a clone when creating `DictionaryArray` from ArrayData [\#9185](https://github.com/apache/arrow-rs/pull/9185) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- fix: take\_run return empty array instead of panic. [\#9182](https://github.com/apache/arrow-rs/pull/9182) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([thorfour](https://github.com/thorfour)) +- lint: remove unused function \(fix clippy [\#9178](https://github.com/apache/arrow-rs/pull/9178) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) +- add `#[inline]` to `BitIterator` `next` function [\#9177](https://github.com/apache/arrow-rs/pull/9177) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) +- Add ListView support to `arrow-row` and `arrow-ord` [\#9176](https://github.com/apache/arrow-rs/pull/9176) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([brancz](https://github.com/brancz)) +- arrow-cast: Add display formatter for ListView [\#9175](https://github.com/apache/arrow-rs/pull/9175) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([brancz](https://github.com/brancz)) +- Add BinaryFormatSupport and Row Encoder to `arrow-avro` Writer [\#9171](https://github.com/apache/arrow-rs/pull/9171) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([jecsand838](https://github.com/jecsand838)) +- docs\(parquet\): move async parquet example into ArrowReaderBuilder docs [\#9167](https://github.com/apache/arrow-rs/pull/9167) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([vigneshsiva11](https://github.com/vigneshsiva11)) +- feat\(array\): add `RecordBatchStream` trait [\#9166](https://github.com/apache/arrow-rs/pull/9166) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([lidavidm](https://github.com/lidavidm)) +- refactor: streamline date64 tests [\#9165](https://github.com/apache/arrow-rs/pull/9165) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([cht42](https://github.com/cht42)) +- docs: update examples in ArrowReaderOptions to use in-memory buffers [\#9163](https://github.com/apache/arrow-rs/pull/9163) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([AndreaBozzo](https://github.com/AndreaBozzo)) +- Add Avro Reader projection API [\#9162](https://github.com/apache/arrow-rs/pull/9162) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([jecsand838](https://github.com/jecsand838)) +- Avoid a clone when creating StringArray/BinaryArray from ArrayData [\#9160](https://github.com/apache/arrow-rs/pull/9160) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- fix missing utf8 check for conversion from BinaryViewArray to StringViewArray [\#9158](https://github.com/apache/arrow-rs/pull/9158) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Minor: try and avoid an allocation creating `GenericByteViewArray` from `ArrayData` [\#9156](https://github.com/apache/arrow-rs/pull/9156) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Add find\_nth\_set\_bit\_position [\#9151](https://github.com/apache/arrow-rs/pull/9151) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan)) +- feat: add null comparison handling in make\_comparator [\#9150](https://github.com/apache/arrow-rs/pull/9150) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Weijun-H](https://github.com/Weijun-H)) +- Uncomment part of test\_utf8\_single\_column\_reader\_test [\#9148](https://github.com/apache/arrow-rs/pull/9148) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([sdf-jkl](https://github.com/sdf-jkl)) +- arrow-ipc: Add tests for nested dicts for Map and Union arrays [\#9146](https://github.com/apache/arrow-rs/pull/9146) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([brancz](https://github.com/brancz)) +- Update ASF copyright year in NOTICE [\#9145](https://github.com/apache/arrow-rs/pull/9145) ([mohit7705](https://github.com/mohit7705)) +- Avoid panic on Date32 overflow [\#9144](https://github.com/apache/arrow-rs/pull/9144) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([cht42](https://github.com/cht42)) +- feat: add `reserve` to `Rows` [\#9142](https://github.com/apache/arrow-rs/pull/9142) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) +- docs\(variant\): fix VariantObject::get documentation to reflect Option return type [\#9139](https://github.com/apache/arrow-rs/pull/9139) ([mohit7705](https://github.com/mohit7705)) +- Add `BooleanBufferBuilder::extend_trusted_len` [\#9137](https://github.com/apache/arrow-rs/pull/9137) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan)) +- fix: support cast from `Null` to list view/run encoded/union types [\#9134](https://github.com/apache/arrow-rs/pull/9134) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Jefffrey](https://github.com/Jefffrey)) +- Fix clippy [\#9130](https://github.com/apache/arrow-rs/pull/9130) ([alamb](https://github.com/alamb)) +- Fix IPC roundtripping dicts nested in ListViews [\#9126](https://github.com/apache/arrow-rs/pull/9126) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([brancz](https://github.com/brancz)) +- Update readme for geospatial crate [\#9124](https://github.com/apache/arrow-rs/pull/9124) ([paleolimbot](https://github.com/paleolimbot)) +- \[Parquet\] perf: Create `PrimitiveArray`s directly rather than via `ArrayData` [\#9122](https://github.com/apache/arrow-rs/pull/9122) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([alamb](https://github.com/alamb)) +- \[Parquet\] perf: Create Utf8/BinaryViewArray directly rather than via `ArrayData` [\#9121](https://github.com/apache/arrow-rs/pull/9121) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([alamb](https://github.com/alamb)) +- \[parquet\] Add row group index virtual column [\#9117](https://github.com/apache/arrow-rs/pull/9117) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([friendlymatthew](https://github.com/friendlymatthew)) +- docs\(parquet\): add example for preserving dictionary encoding [\#9116](https://github.com/apache/arrow-rs/pull/9116) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([AndreaBozzo](https://github.com/AndreaBozzo)) +- doc: add example of RowFilter usage [\#9115](https://github.com/apache/arrow-rs/pull/9115) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([sonhmai](https://github.com/sonhmai)) +- docs: Update release schedule in README.md [\#9111](https://github.com/apache/arrow-rs/pull/9111) ([alamb](https://github.com/alamb)) +- feat: add benchmarks for json parser [\#9107](https://github.com/apache/arrow-rs/pull/9107) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Weijun-H](https://github.com/Weijun-H)) +- chore: switch test from `bincode` to maintained `postcard` crate \(RUSTSEC-2025-0141 \) [\#9104](https://github.com/apache/arrow-rs/pull/9104) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Add nullif\_kernel benchmark [\#9089](https://github.com/apache/arrow-rs/pull/9089) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- \[Variant\] Support Shredded Lists/Array in `variant_get` [\#9049](https://github.com/apache/arrow-rs/pull/9049) ([liamzwbao](https://github.com/liamzwbao)) +- fix:\[9018\]Fixed RunArray slice offsets [\#9036](https://github.com/apache/arrow-rs/pull/9036) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([manishkr](https://github.com/manishkr)) +- Support Float16 for create\_random\_array [\#9029](https://github.com/apache/arrow-rs/pull/9029) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([niebayes](https://github.com/niebayes)) +- fix: display `0 secs` for empty DayTime/MonthDayNano intervals [\#9023](https://github.com/apache/arrow-rs/pull/9023) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Jefffrey](https://github.com/Jefffrey)) +- Add options to skip decoding `Statistics` and `SizeStatistics` in Parquet metadata [\#9008](https://github.com/apache/arrow-rs/pull/9008) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://github.com/etseidl)) + ## [57.3.0](https://github.com/apache/arrow-rs/tree/57.3.0) (2026-02-02) [Full Changelog](https://github.com/apache/arrow-rs/compare/57.2.0...57.3.0) diff --git a/CHANGELOG.md b/CHANGELOG.md index e73ee8ba3356..baccdfa79cbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,200 +19,123 @@ # Changelog -## [58.0.0](https://github.com/apache/arrow-rs/tree/58.0.0) (2026-02-19) +## [58.1.0](https://github.com/apache/arrow-rs/tree/58.1.0) (2026-03-20) -[Full Changelog](https://github.com/apache/arrow-rs/compare/57.3.0...58.0.0) - -**Breaking changes:** - -- Remove support for List types in bit\_length kernel [\#9350](https://github.com/apache/arrow-rs/pull/9350) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([codephage2020](https://github.com/codephage2020)) -- Optimize `from_bitwise_unary_op` [\#9297](https://github.com/apache/arrow-rs/pull/9297) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan)) -- Mark `BufferBuilder::new_from_buffer` as unsafe [\#9292](https://github.com/apache/arrow-rs/pull/9292) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Jefffrey](https://github.com/Jefffrey)) -- \[Variant\] Support `['fieldName']` in VariantPath parser [\#9276](https://github.com/apache/arrow-rs/pull/9276) ([klion26](https://github.com/klion26)) -- Remove parquet arrow\_cast dependency [\#9077](https://github.com/apache/arrow-rs/pull/9077) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([tustvold](https://github.com/tustvold)) -- feat: change default behavior for Parquet `PageEncodingStats` to bitmask [\#9051](https://github.com/apache/arrow-rs/pull/9051) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([WaterWhisperer](https://github.com/WaterWhisperer)) -- \[arrow\] Minimize allocation in GenericViewArray::slice\(\) [\#9016](https://github.com/apache/arrow-rs/pull/9016) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([maxburke](https://github.com/maxburke)) +[Full Changelog](https://github.com/apache/arrow-rs/compare/58.0.0...58.1.0) **Implemented enhancements:** -- Avoid allocating a `Vec` in `StructBuilder` [\#9427](https://github.com/apache/arrow-rs/issues/9427) -- Zstd context reuse [\#9401](https://github.com/apache/arrow-rs/issues/9401) -- Optimize `from_bitwise_unary_op` [\#9364](https://github.com/apache/arrow-rs/issues/9364) -- Support `RunEndEncoded` in ord comparator [\#9360](https://github.com/apache/arrow-rs/issues/9360) -- Support `RunEndEncoded` arrays in `arrow-json` [\#9359](https://github.com/apache/arrow-rs/issues/9359) -- Support `BinaryView` in `bit_length` kernel [\#9351](https://github.com/apache/arrow-rs/issues/9351) -- Remove support for `List` types in `bit_length` kernel [\#9349](https://github.com/apache/arrow-rs/issues/9349) -- Support roundtrip `ListView` in parquet arrow writer [\#9344](https://github.com/apache/arrow-rs/issues/9344) -- Support `ListView` in `length` kernel [\#9343](https://github.com/apache/arrow-rs/issues/9343) -- Support `ListView` in sort kernel [\#9341](https://github.com/apache/arrow-rs/issues/9341) -- Add some way to create a Timestamp from a `DateTime` [\#9337](https://github.com/apache/arrow-rs/issues/9337) -- Introduce `DataType::is_list` and `DataType::IsBinary` [\#9326](https://github.com/apache/arrow-rs/issues/9326) -- Performance of creating all null dictionary array can be improved [\#9321](https://github.com/apache/arrow-rs/issues/9321) -- \[arrow-avro\] Add missing Arrow DataType support with `avro_custom_types` round-trip + non-custom fallbacks [\#9290](https://github.com/apache/arrow-rs/issues/9290) +- Reuse compression dict lz4\_block [\#9566](https://github.com/apache/arrow-rs/issues/9566) +- \[Variant\] Add `variant_to_arrow` `Struct` type support [\#9529](https://github.com/apache/arrow-rs/issues/9529) +- \[Variant\] Add `unshred_variant` support for `Binary` and `LargeBinary` types [\#9526](https://github.com/apache/arrow-rs/issues/9526) +- \[Variant\] Add `shred_variant` support for `LargeUtf8` and `LargeBinary` types [\#9525](https://github.com/apache/arrow-rs/issues/9525) +- \[Variant\] `variant_get` tests clean up [\#9517](https://github.com/apache/arrow-rs/issues/9517) +- parquet\_variant: Support LargeUtf8 typed value in `unshred_variant` [\#9513](https://github.com/apache/arrow-rs/issues/9513) +- parquet-variant: Support string view typed value in `unshred_variant` [\#9512](https://github.com/apache/arrow-rs/issues/9512) +- Deprecate ArrowTimestampType::make\_value in favor of from\_naive\_datetime [\#9490](https://github.com/apache/arrow-rs/issues/9490) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Followup for support \['fieldName'\] in VariantPath [\#9478](https://github.com/apache/arrow-rs/issues/9478) +- Speedup DELTA\_BINARY\_PACKED decoding when bitwidth is 0 [\#9476](https://github.com/apache/arrow-rs/issues/9476) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Support CSV files encoded with charsets other than UTF-8 [\#9465](https://github.com/apache/arrow-rs/issues/9465) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Expose Avro writer schema when building the reader [\#9460](https://github.com/apache/arrow-rs/issues/9460) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Python: avoid importing pyarrow classes ever time [\#9438](https://github.com/apache/arrow-rs/issues/9438) +- Add `append_nulls` to `MapBuilder` [\#9431](https://github.com/apache/arrow-rs/issues/9431) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Add `append_non_nulls` to `StructBuilder` [\#9429](https://github.com/apache/arrow-rs/issues/9429) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Add `append_value_n` to GenericByteBuilder [\#9425](https://github.com/apache/arrow-rs/issues/9425) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Optimize `from_bitwise_binary_op` [\#9378](https://github.com/apache/arrow-rs/issues/9378) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Configurable Arrow representation of UTC timestamps for Avro reader [\#9279](https://github.com/apache/arrow-rs/issues/9279) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] **Fixed bugs:** -- ArrowArrayStreamReader errors on zero-column record batches [\#9394](https://github.com/apache/arrow-rs/issues/9394) -- Regression on main \(58\): Parquet argument error: Parquet error: Required field type\_ is missing [\#9315](https://github.com/apache/arrow-rs/issues/9315) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- MutableArrayData::extend does not copy child values for ListView arrays [\#9561](https://github.com/apache/arrow-rs/issues/9561) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- ListView interleave bug [\#9559](https://github.com/apache/arrow-rs/issues/9559) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Flight encoding panics with "no dict id for field" with nested dict arrays [\#9555](https://github.com/apache/arrow-rs/issues/9555) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] [[arrow-flight](https://github.com/apache/arrow-rs/labels/arrow-flight)] +- "DeltaBitPackDecoder only supports Int32Type and Int64Type" but unsigned types are supported too [\#9551](https://github.com/apache/arrow-rs/issues/9551) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Potential overflow when calling `util::bit_mask::set_bits` \(soundness issue\) [\#9543](https://github.com/apache/arrow-rs/issues/9543) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- handle Null type in try\_merge for Struct, List, LargeList, and Union [\#9523](https://github.com/apache/arrow-rs/issues/9523) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Invalid offset in sparse column chunk data for multiple predicates [\#9516](https://github.com/apache/arrow-rs/issues/9516) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- debug\_assert\_eq! in BatchCoalescer panics in debug mode when batch\_size \< 4 [\#9506](https://github.com/apache/arrow-rs/issues/9506) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Parquet Statistics::null\_count\_opt wrongly returns Some\(0\) when stats are missing [\#9451](https://github.com/apache/arrow-rs/issues/9451) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Error "Not all children array length are the same!" when decoding rows spanning across page boundaries in parquet file when using `RowSelection` [\#9370](https://github.com/apache/arrow-rs/issues/9370) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Avro schema resolution not properly supported for complex types [\#9336](https://github.com/apache/arrow-rs/issues/9336) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] **Documentation updates:** -- Improve safety documentation of the `Array` trait [\#9314](https://github.com/apache/arrow-rs/pull/9314) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Improve docs and add build\(\) method to `{Null,Boolean,}BufferBuilder` [\#9155](https://github.com/apache/arrow-rs/pull/9155) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Improve `ArrowReaderBuilder::with_row_filter` documentation [\#9153](https://github.com/apache/arrow-rs/pull/9153) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([alamb](https://github.com/alamb)) -- docs: Improve main README.md and highlight community [\#9119](https://github.com/apache/arrow-rs/pull/9119) ([alamb](https://github.com/alamb)) -- Docs: Add additional documentation and example for `make_array` [\#9112](https://github.com/apache/arrow-rs/pull/9112) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- doc: fix link on FixedSizeListArray doc [\#9033](https://github.com/apache/arrow-rs/pull/9033) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Jefffrey](https://github.com/Jefffrey)) +- Update planned release schedule in README.md [\#9466](https://github.com/apache/arrow-rs/pull/9466) ([alamb](https://github.com/alamb)) **Performance improvements:** -- Replace `ArrayData` with direct Array construction [\#9338](https://github.com/apache/arrow-rs/pull/9338) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) -- Remove some `unsafe` and allocations when creating PrimitiveArrays from Vec and `from_trusted_len_iter` [\#9299](https://github.com/apache/arrow-rs/pull/9299) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- parquet: rle skip decode loop when batch contains all max levels \(aka no nulls\) [\#9258](https://github.com/apache/arrow-rs/pull/9258) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([lyang24](https://github.com/lyang24)) -- Improve parquet BinaryView / StringView decoder performance \(up to -35%\) [\#9236](https://github.com/apache/arrow-rs/pull/9236) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) -- Avoid a clone when creating `BooleanArray` from ArrayData [\#9159](https://github.com/apache/arrow-rs/pull/9159) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Avoid overallocating arrays in coalesce primitives / views [\#9132](https://github.com/apache/arrow-rs/pull/9132) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan)) -- perf: Avoid ArrayData allocation in PrimitiveArray::reinterpret\_cast [\#9129](https://github.com/apache/arrow-rs/pull/9129) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- \[Parquet\] perf: Create StructArrays directly rather than via `ArrayData` \(1% improvement\) [\#9120](https://github.com/apache/arrow-rs/pull/9120) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Avoid clones in `make_array` for `StructArray` and `GenericByteViewArray` [\#9114](https://github.com/apache/arrow-rs/pull/9114) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- perf: optimize hex decoding in json \(1.8x faster in binary-heavy\) [\#9091](https://github.com/apache/arrow-rs/pull/9091) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Weijun-H](https://github.com/Weijun-H)) -- Speed up binary kernels \(30% faster `and` and `or`\), add `BooleanBuffer::from_bitwise_binary_op` [\#9090](https://github.com/apache/arrow-rs/pull/9090) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- perf: improve field indexing in JSON StructArrayDecoder \(1.7x speed up\) [\#9086](https://github.com/apache/arrow-rs/pull/9086) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Weijun-H](https://github.com/Weijun-H)) -- bench: added to row\_format benchmark conversion of 53 non-nested columns [\#9081](https://github.com/apache/arrow-rs/pull/9081) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) -- perf: improve calculating length performance for view byte array in row conversion [\#9080](https://github.com/apache/arrow-rs/pull/9080) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) -- perf: improve calculating length performance for nested arrays in row conversion [\#9079](https://github.com/apache/arrow-rs/pull/9079) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) -- perf: improve calculating length performance for `GenericByteArray` in row conversion [\#9078](https://github.com/apache/arrow-rs/pull/9078) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) +- Introduce `NullBuffer::try_from_unsliced` to simplify array construction [\#9385](https://github.com/apache/arrow-rs/issues/9385) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- perf: Coalesce page fetches when RowSelection selects all rows [\#9578](https://github.com/apache/arrow-rs/pull/9578) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) +- Use chunks\_exact for has\_true/has\_false to enable compiler unrolling [\#9570](https://github.com/apache/arrow-rs/pull/9570) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([adriangb](https://github.com/adriangb)) +- pyarrow: Cache the imported classes to avoid importing them each time [\#9439](https://github.com/apache/arrow-rs/pull/9439) ([Tpt](https://github.com/Tpt)) **Closed issues:** -- BatchCoalescer::push\_batch panics on schema mismatch instead of returning error [\#9389](https://github.com/apache/arrow-rs/issues/9389) -- Release arrow-rs / parquet Minor version `57.3.0` \(January 2026\) [\#9240](https://github.com/apache/arrow-rs/issues/9240) -- \[Variant\] support `..` and `['fieldName']` syntax in the VariantPath parser [\#9050](https://github.com/apache/arrow-rs/issues/9050) -- Support Float16 for create\_random\_array [\#9028](https://github.com/apache/arrow-rs/issues/9028) +- Duplicate macro definition: `partially_shredded_variant_array_gen` [\#9492](https://github.com/apache/arrow-rs/issues/9492) +- Enable `LargeList` / `ListView` / `LargeListView` for `VariantArray::try_new` [\#9455](https://github.com/apache/arrow-rs/issues/9455) +- Support variables/expressions in record\_batch! macro [\#9245](https://github.com/apache/arrow-rs/issues/9245) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] **Merged pull requests:** -- Avoid allocating a `Vec` in `StructBuilder` [\#9428](https://github.com/apache/arrow-rs/pull/9428) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Fokko](https://github.com/Fokko)) -- fix: fixed trait functions clash get\_date\_time\_part\_extract\_fn \(\#8221\) [\#9424](https://github.com/apache/arrow-rs/pull/9424) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([esavier](https://github.com/esavier)) -- \[Minor\] Use per-predicate projection masks in arrow\_reader\_clickbench benchmark [\#9413](https://github.com/apache/arrow-rs/pull/9413) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) -- Fix `ArrowArrayStreamReader` for 0-columns record batch streams [\#9405](https://github.com/apache/arrow-rs/pull/9405) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([jonded94](https://github.com/jonded94)) -- Use zstd::bulk API in IPC and Parquet with context reuse for compression and decompression [\#9400](https://github.com/apache/arrow-rs/pull/9400) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan)) -- Reproduce the issue of \#9370 in a minimal, end-to-end way [\#9399](https://github.com/apache/arrow-rs/pull/9399) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([jonded94](https://github.com/jonded94)) -- perf: optimize skipper for varint values used when projecting Avro record types [\#9397](https://github.com/apache/arrow-rs/pull/9397) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mzabaluev](https://github.com/mzabaluev)) -- fix: return error instead of panic on schema mismatch in BatchCoalescer::push\_batch [\#9390](https://github.com/apache/arrow-rs/pull/9390) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([bvolpato-dd](https://github.com/bvolpato-dd)) -- Minor: Add additional test coverage for WriterProperties::{max\_row\_group\_row\_count,max\_row\_group\_size} [\#9387](https://github.com/apache/arrow-rs/pull/9387) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([alamb](https://github.com/alamb)) -- Moving invalid\_utf8 tests into a separate mod [\#9384](https://github.com/apache/arrow-rs/pull/9384) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([sdf-jkl](https://github.com/sdf-jkl)) -- Update sysinfo requirement from 0.37.1 to 0.38.1 [\#9383](https://github.com/apache/arrow-rs/pull/9383) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([dependabot[bot]](https://github.com/apps/dependabot)) -- feat: support RunEndEncoded arrays in arrow-json reader and writer [\#9379](https://github.com/apache/arrow-rs/pull/9379) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Abhisheklearn12](https://github.com/Abhisheklearn12)) -- Remove lint issues in parquet-related code. [\#9375](https://github.com/apache/arrow-rs/pull/9375) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([brunal](https://github.com/brunal)) -- Add RunEndEncoded array comparator [\#9368](https://github.com/apache/arrow-rs/pull/9368) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([codephage2020](https://github.com/codephage2020)) -- feat: support BinaryView in bit\_length kernel [\#9363](https://github.com/apache/arrow-rs/pull/9363) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Abhisheklearn12](https://github.com/Abhisheklearn12)) -- Add regression tests for Parquet large binary offset overflow [\#9361](https://github.com/apache/arrow-rs/pull/9361) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([vigneshsiva11](https://github.com/vigneshsiva11)) -- feat: add max\_row\_group\_bytes option to WriterProperties [\#9357](https://github.com/apache/arrow-rs/pull/9357) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([yonipeleg33](https://github.com/yonipeleg33)) -- doc: remove disclaimer about `ListView` not being fully supported [\#9356](https://github.com/apache/arrow-rs/pull/9356) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Jefffrey](https://github.com/Jefffrey)) -- Move row\_filter async tests from parquet async reader [\#9355](https://github.com/apache/arrow-rs/pull/9355) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([sdf-jkl](https://github.com/sdf-jkl)) -- \[Parquet\] Allow setting page size per column [\#9353](https://github.com/apache/arrow-rs/pull/9353) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([XiangpengHao](https://github.com/XiangpengHao)) -- feat: Support roundtrip ListView in parquet arrow writer [\#9352](https://github.com/apache/arrow-rs/pull/9352) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([codephage2020](https://github.com/codephage2020)) -- feat: add ListView and LargeListView support to arrow-ord [\#9347](https://github.com/apache/arrow-rs/pull/9347) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([codephage2020](https://github.com/codephage2020)) -- Support ListView in length kernel [\#9346](https://github.com/apache/arrow-rs/pull/9346) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([vegarsti](https://github.com/vegarsti)) -- feat: Add from\_datetime method to Timestamp types [\#9345](https://github.com/apache/arrow-rs/pull/9345) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([codephage2020](https://github.com/codephage2020)) -- \[main\] Update version to 57.3.0, add changelog [\#9334](https://github.com/apache/arrow-rs/pull/9334) ([alamb](https://github.com/alamb)) -- build\(deps\): update pyo3 requirement from 0.27.1 to 0.28.0 [\#9331](https://github.com/apache/arrow-rs/pull/9331) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Add `DataType::is_list` and `DataType::is_binary` [\#9327](https://github.com/apache/arrow-rs/pull/9327) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([AdamGS](https://github.com/AdamGS)) -- Fix string array equality when the values buffer is the same and only the offsets to access it differ [\#9325](https://github.com/apache/arrow-rs/pull/9325) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([jhorstmann](https://github.com/jhorstmann)) -- perf: skip validation of dictionary keys if all null [\#9322](https://github.com/apache/arrow-rs/pull/9322) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([albertlockett](https://github.com/albertlockett)) -- parquet: use rwlock instead of mutex in predicate cache [\#9319](https://github.com/apache/arrow-rs/pull/9319) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([lyang24](https://github.com/lyang24)) -- nit: remove usused code [\#9318](https://github.com/apache/arrow-rs/pull/9318) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([lyang24](https://github.com/lyang24)) -- Remove unnecessary Arc\ [\#9316](https://github.com/apache/arrow-rs/pull/9316) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([scovich](https://github.com/scovich)) -- Optimize data page statistics conversion \(up to 4x\) [\#9303](https://github.com/apache/arrow-rs/pull/9303) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan)) -- \[regression\] Error with adaptive predicate pushdown: "Invalid offset in sparse column chunk data: 754, no matching page found." [\#9301](https://github.com/apache/arrow-rs/pull/9301) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([sdf-jkl](https://github.com/sdf-jkl)) -- Improve `PrimitiveArray::from_iter` perf [\#9294](https://github.com/apache/arrow-rs/pull/9294) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan)) -- Add additional Arrow type support [\#9291](https://github.com/apache/arrow-rs/pull/9291) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([jecsand838](https://github.com/jecsand838)) -- fix: ensure `BufferBuilder::truncate` doesn't overset length [\#9288](https://github.com/apache/arrow-rs/pull/9288) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Jefffrey](https://github.com/Jefffrey)) -- Add benchmark for row group index reader perf [\#9285](https://github.com/apache/arrow-rs/pull/9285) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([friendlymatthew](https://github.com/friendlymatthew)) -- fix union array row converter to handle non-sequential type ids [\#9283](https://github.com/apache/arrow-rs/pull/9283) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([friendlymatthew](https://github.com/friendlymatthew)) -- parquet: reduce clone in delta byte array decoder [\#9282](https://github.com/apache/arrow-rs/pull/9282) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([lyang24](https://github.com/lyang24)) -- fix: fix \[\[NULL\]\] array doesn't roundtrip in arrow-row bug [\#9275](https://github.com/apache/arrow-rs/pull/9275) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([lichuang](https://github.com/lichuang)) -- Enhance list casting, adding more cases for list views [\#9274](https://github.com/apache/arrow-rs/pull/9274) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Jefffrey](https://github.com/Jefffrey)) -- \[Variant\] Add path index access tests for list [\#9273](https://github.com/apache/arrow-rs/pull/9273) ([liamzwbao](https://github.com/liamzwbao)) -- Factor out json reader's static make\_decoder args to a struct [\#9271](https://github.com/apache/arrow-rs/pull/9271) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([scovich](https://github.com/scovich)) -- make\_decoder accepts borrowed DataType instead of owned [\#9270](https://github.com/apache/arrow-rs/pull/9270) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([scovich](https://github.com/scovich)) -- Implement a more generic from\_nested\_iter method for list arrays [\#9268](https://github.com/apache/arrow-rs/pull/9268) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([jhorstmann](https://github.com/jhorstmann)) -- Move extension type construction logic out of Field [\#9266](https://github.com/apache/arrow-rs/pull/9266) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([scovich](https://github.com/scovich)) -- fix: support casting string to f16 [\#9262](https://github.com/apache/arrow-rs/pull/9262) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Jefffrey](https://github.com/Jefffrey)) -- Add additional coverage for StringViewArray comparisons [\#9257](https://github.com/apache/arrow-rs/pull/9257) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Upgrade to object store 0.13.1 [\#9256](https://github.com/apache/arrow-rs/pull/9256) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) -- \[Parquet\] test adaptive predicate pushdown with skipped page [\#9251](https://github.com/apache/arrow-rs/pull/9251) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([sdf-jkl](https://github.com/sdf-jkl)) -- Speed up string view comparison \(up to 3x\) [\#9250](https://github.com/apache/arrow-rs/pull/9250) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan)) -- Add bench for LocalFileSystem [\#9248](https://github.com/apache/arrow-rs/pull/9248) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) -- \[Parquet\] Add test for reading/writing long UTF8 StringViews [\#9246](https://github.com/apache/arrow-rs/pull/9246) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([alamb](https://github.com/alamb)) -- \[Parquet\] test adaptive predicate pushdown with skipped page [\#9243](https://github.com/apache/arrow-rs/pull/9243) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([erratic-pattern](https://github.com/erratic-pattern)) -- Add tests and fixes for schema resolution bug [\#9237](https://github.com/apache/arrow-rs/pull/9237) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([jecsand838](https://github.com/jecsand838)) -- Revert "Seal Array trait \(\#9092\)", mark `Array` as `unsafe` [\#9234](https://github.com/apache/arrow-rs/pull/9234) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([gabotechs](https://github.com/gabotechs)) -- Speedup filter \(up to ~1.5x\) `FilterBuilder::Optimize`/`BitIndexIterator`/`iter_set_bits_rev` [\#9229](https://github.com/apache/arrow-rs/pull/9229) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan)) -- optimize `RowGroupIndexReader` for single row group reads [\#9226](https://github.com/apache/arrow-rs/pull/9226) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([friendlymatthew](https://github.com/friendlymatthew)) -- test: improve arrow-row fuzz tests [\#9222](https://github.com/apache/arrow-rs/pull/9222) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) -- arrow-cast: support packing to Dictionary\(\_, Utf8View/BinaryView\) [\#9220](https://github.com/apache/arrow-rs/pull/9220) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([ethan-tyler](https://github.com/ethan-tyler)) -- Add additional test coverage for `BatchCoalescer` push\_batch\_with\_filter [\#9218](https://github.com/apache/arrow-rs/pull/9218) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- \[Parquet\] Optimize appending max level comparison in DefinitionLevelDecoder [\#9217](https://github.com/apache/arrow-rs/pull/9217) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([jhorstmann](https://github.com/jhorstmann)) -- Remove dead code to fix clippy failure on main [\#9215](https://github.com/apache/arrow-rs/pull/9215) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- \[Parquet\] perf: reuse seeked File clone in ChunkReader::get\_read\(\) [\#9214](https://github.com/apache/arrow-rs/pull/9214) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([fvaleye](https://github.com/fvaleye)) -- fix: \[9018\]Fixed RunArray slice offsets\(row, cast, eq\) [\#9213](https://github.com/apache/arrow-rs/pull/9213) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([manishkr](https://github.com/manishkr)) -- Add benchmarks for reading struct arrays from parquet [\#9210](https://github.com/apache/arrow-rs/pull/9210) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([jhorstmann](https://github.com/jhorstmann)) -- Support casting negative scale decimals to numeric [\#9207](https://github.com/apache/arrow-rs/pull/9207) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Chiicake](https://github.com/Chiicake)) -- Deprecate `ArrowReaderOptions::with_page_index` and update API [\#9199](https://github.com/apache/arrow-rs/pull/9199) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([friendlymatthew](https://github.com/friendlymatthew)) -- arrow-ipc: add reset method to DictionaryTracker [\#9196](https://github.com/apache/arrow-rs/pull/9196) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([albertlockett](https://github.com/albertlockett)) -- Avoid a clone when creating `ListArray` from ArrayData [\#9194](https://github.com/apache/arrow-rs/pull/9194) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Avoid a clone when creating `ListViewArray` from ArrayData [\#9193](https://github.com/apache/arrow-rs/pull/9193) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Avoid a clone when creating `MapArray` from ArrayData [\#9192](https://github.com/apache/arrow-rs/pull/9192) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Avoid a clone when creating `NullArray` from ArrayData [\#9191](https://github.com/apache/arrow-rs/pull/9191) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Avoid a clone when creating `PrimitiveArray` from ArrayData [\#9190](https://github.com/apache/arrow-rs/pull/9190) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Avoid a clone when creating `RunEndArray` from ArrayData [\#9189](https://github.com/apache/arrow-rs/pull/9189) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Avoid a clone when creating `UnionArray` from ArrayData [\#9188](https://github.com/apache/arrow-rs/pull/9188) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Avoid a clone when creating `FixedSizeListArray` from ArrayData [\#9187](https://github.com/apache/arrow-rs/pull/9187) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Avoid a clone when creating `FixedSizeBinaryArray` from ArrayData [\#9186](https://github.com/apache/arrow-rs/pull/9186) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Avoid a clone when creating `DictionaryArray` from ArrayData [\#9185](https://github.com/apache/arrow-rs/pull/9185) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- fix: take\_run return empty array instead of panic. [\#9182](https://github.com/apache/arrow-rs/pull/9182) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([thorfour](https://github.com/thorfour)) -- lint: remove unused function \(fix clippy [\#9178](https://github.com/apache/arrow-rs/pull/9178) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) -- add `#[inline]` to `BitIterator` `next` function [\#9177](https://github.com/apache/arrow-rs/pull/9177) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) -- Add ListView support to `arrow-row` and `arrow-ord` [\#9176](https://github.com/apache/arrow-rs/pull/9176) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([brancz](https://github.com/brancz)) -- arrow-cast: Add display formatter for ListView [\#9175](https://github.com/apache/arrow-rs/pull/9175) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([brancz](https://github.com/brancz)) -- Add BinaryFormatSupport and Row Encoder to `arrow-avro` Writer [\#9171](https://github.com/apache/arrow-rs/pull/9171) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([jecsand838](https://github.com/jecsand838)) -- docs\(parquet\): move async parquet example into ArrowReaderBuilder docs [\#9167](https://github.com/apache/arrow-rs/pull/9167) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([vigneshsiva11](https://github.com/vigneshsiva11)) -- feat\(array\): add `RecordBatchStream` trait [\#9166](https://github.com/apache/arrow-rs/pull/9166) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([lidavidm](https://github.com/lidavidm)) -- refactor: streamline date64 tests [\#9165](https://github.com/apache/arrow-rs/pull/9165) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([cht42](https://github.com/cht42)) -- docs: update examples in ArrowReaderOptions to use in-memory buffers [\#9163](https://github.com/apache/arrow-rs/pull/9163) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([AndreaBozzo](https://github.com/AndreaBozzo)) -- Add Avro Reader projection API [\#9162](https://github.com/apache/arrow-rs/pull/9162) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([jecsand838](https://github.com/jecsand838)) -- Avoid a clone when creating StringArray/BinaryArray from ArrayData [\#9160](https://github.com/apache/arrow-rs/pull/9160) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- fix missing utf8 check for conversion from BinaryViewArray to StringViewArray [\#9158](https://github.com/apache/arrow-rs/pull/9158) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Minor: try and avoid an allocation creating `GenericByteViewArray` from `ArrayData` [\#9156](https://github.com/apache/arrow-rs/pull/9156) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Add find\_nth\_set\_bit\_position [\#9151](https://github.com/apache/arrow-rs/pull/9151) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan)) -- feat: add null comparison handling in make\_comparator [\#9150](https://github.com/apache/arrow-rs/pull/9150) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Weijun-H](https://github.com/Weijun-H)) -- Uncomment part of test\_utf8\_single\_column\_reader\_test [\#9148](https://github.com/apache/arrow-rs/pull/9148) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([sdf-jkl](https://github.com/sdf-jkl)) -- arrow-ipc: Add tests for nested dicts for Map and Union arrays [\#9146](https://github.com/apache/arrow-rs/pull/9146) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([brancz](https://github.com/brancz)) -- Update ASF copyright year in NOTICE [\#9145](https://github.com/apache/arrow-rs/pull/9145) ([mohit7705](https://github.com/mohit7705)) -- Avoid panic on Date32 overflow [\#9144](https://github.com/apache/arrow-rs/pull/9144) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([cht42](https://github.com/cht42)) -- feat: add `reserve` to `Rows` [\#9142](https://github.com/apache/arrow-rs/pull/9142) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) -- docs\(variant\): fix VariantObject::get documentation to reflect Option return type [\#9139](https://github.com/apache/arrow-rs/pull/9139) ([mohit7705](https://github.com/mohit7705)) -- Add `BooleanBufferBuilder::extend_trusted_len` [\#9137](https://github.com/apache/arrow-rs/pull/9137) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan)) -- fix: support cast from `Null` to list view/run encoded/union types [\#9134](https://github.com/apache/arrow-rs/pull/9134) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Jefffrey](https://github.com/Jefffrey)) -- Fix clippy [\#9130](https://github.com/apache/arrow-rs/pull/9130) ([alamb](https://github.com/alamb)) -- Fix IPC roundtripping dicts nested in ListViews [\#9126](https://github.com/apache/arrow-rs/pull/9126) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([brancz](https://github.com/brancz)) -- Update readme for geospatial crate [\#9124](https://github.com/apache/arrow-rs/pull/9124) ([paleolimbot](https://github.com/paleolimbot)) -- \[Parquet\] perf: Create `PrimitiveArray`s directly rather than via `ArrayData` [\#9122](https://github.com/apache/arrow-rs/pull/9122) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([alamb](https://github.com/alamb)) -- \[Parquet\] perf: Create Utf8/BinaryViewArray directly rather than via `ArrayData` [\#9121](https://github.com/apache/arrow-rs/pull/9121) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([alamb](https://github.com/alamb)) -- \[parquet\] Add row group index virtual column [\#9117](https://github.com/apache/arrow-rs/pull/9117) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([friendlymatthew](https://github.com/friendlymatthew)) -- docs\(parquet\): add example for preserving dictionary encoding [\#9116](https://github.com/apache/arrow-rs/pull/9116) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([AndreaBozzo](https://github.com/AndreaBozzo)) -- doc: add example of RowFilter usage [\#9115](https://github.com/apache/arrow-rs/pull/9115) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([sonhmai](https://github.com/sonhmai)) -- docs: Update release schedule in README.md [\#9111](https://github.com/apache/arrow-rs/pull/9111) ([alamb](https://github.com/alamb)) -- feat: add benchmarks for json parser [\#9107](https://github.com/apache/arrow-rs/pull/9107) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Weijun-H](https://github.com/Weijun-H)) -- chore: switch test from `bincode` to maintained `postcard` crate \(RUSTSEC-2025-0141 \) [\#9104](https://github.com/apache/arrow-rs/pull/9104) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Add nullif\_kernel benchmark [\#9089](https://github.com/apache/arrow-rs/pull/9089) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- \[Variant\] Support Shredded Lists/Array in `variant_get` [\#9049](https://github.com/apache/arrow-rs/pull/9049) ([liamzwbao](https://github.com/liamzwbao)) -- fix:\[9018\]Fixed RunArray slice offsets [\#9036](https://github.com/apache/arrow-rs/pull/9036) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([manishkr](https://github.com/manishkr)) -- Support Float16 for create\_random\_array [\#9029](https://github.com/apache/arrow-rs/pull/9029) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([niebayes](https://github.com/niebayes)) -- fix: display `0 secs` for empty DayTime/MonthDayNano intervals [\#9023](https://github.com/apache/arrow-rs/pull/9023) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Jefffrey](https://github.com/Jefffrey)) -- Add options to skip decoding `Statistics` and `SizeStatistics` in Parquet metadata [\#9008](https://github.com/apache/arrow-rs/pull/9008) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://github.com/etseidl)) +- \[Variant\] Add unshred\_variant support for Binary and LargeBinary types [\#9576](https://github.com/apache/arrow-rs/pull/9576) ([kunalsinghdadhwal](https://github.com/kunalsinghdadhwal)) +- \[Variant\] Add `variant_to_arrow` `Struct` type support [\#9572](https://github.com/apache/arrow-rs/pull/9572) ([sdf-jkl](https://github.com/sdf-jkl)) +- Make Sbbf Constructers Public [\#9569](https://github.com/apache/arrow-rs/pull/9569) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([cetra3](https://github.com/cetra3)) +- fix: Used `checked_add` for bounds checks to avoid UB [\#9568](https://github.com/apache/arrow-rs/pull/9568) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([etseidl](https://github.com/etseidl)) +- Add mutable operations to BooleanBuffer \(Bit\*Assign\) [\#9567](https://github.com/apache/arrow-rs/pull/9567) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan)) +- chore\(deps\): update lz4\_flex requirement from 0.12 to 0.13 [\#9565](https://github.com/apache/arrow-rs/pull/9565) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([dependabot[bot]](https://github.com/apps/dependabot)) +- arrow-select: fix MutableArrayData interleave for ListView [\#9560](https://github.com/apache/arrow-rs/pull/9560) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([asubiotto](https://github.com/asubiotto)) +- Move `ValueIter` into own module, and add public `record_count` function [\#9557](https://github.com/apache/arrow-rs/pull/9557) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Rafferty97](https://github.com/Rafferty97)) +- arrow-flight: generate dict\_ids for dicts nested inside complex types [\#9556](https://github.com/apache/arrow-rs/pull/9556) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] [[arrow-flight](https://github.com/apache/arrow-rs/labels/arrow-flight)] ([asubiotto](https://github.com/asubiotto)) +- add `shred_variant` support for `LargeUtf8` and `LargeBinary` [\#9554](https://github.com/apache/arrow-rs/pull/9554) ([sdf-jkl](https://github.com/sdf-jkl)) +- \[minor\] Download clickbench file when missing [\#9553](https://github.com/apache/arrow-rs/pull/9553) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) +- DeltaBitPackEncoderConversion: Fix panic message on invalid type [\#9552](https://github.com/apache/arrow-rs/pull/9552) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([progval](https://github.com/progval)) +- Replace interleave overflow panic with error [\#9549](https://github.com/apache/arrow-rs/pull/9549) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([xudong963](https://github.com/xudong963)) +- feat\(arrow-avro\): `HeaderInfo` to expose OCF header [\#9548](https://github.com/apache/arrow-rs/pull/9548) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mzabaluev](https://github.com/mzabaluev)) +- chore: Protect `main` branch with required reviews [\#9547](https://github.com/apache/arrow-rs/pull/9547) ([comphead](https://github.com/comphead)) +- Add benchmark for `infer_json_schema` [\#9546](https://github.com/apache/arrow-rs/pull/9546) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Rafferty97](https://github.com/Rafferty97)) +- chore\(deps\): bump black from 24.3.0 to 26.3.1 in /parquet/pytest [\#9545](https://github.com/apache/arrow-rs/pull/9545) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([dependabot[bot]](https://github.com/apps/dependabot)) +- Unroll interleave -25-30% [\#9542](https://github.com/apache/arrow-rs/pull/9542) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Dandandan](https://github.com/Dandandan)) +- Optimize `take_fixed_size_binary` For Predefined Value Lengths [\#9535](https://github.com/apache/arrow-rs/pull/9535) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([tobixdev](https://github.com/tobixdev)) +- feat: expose arrow schema on async avro reader [\#9534](https://github.com/apache/arrow-rs/pull/9534) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mzabaluev](https://github.com/mzabaluev)) +- Make with\_file\_decryption\_properties pub instead of pub\(crate\) [\#9532](https://github.com/apache/arrow-rs/pull/9532) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) +- fix: handle Null type in try\_merge for Struct, List, LargeList, and Union [\#9524](https://github.com/apache/arrow-rs/pull/9524) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([zhuqi-lucas](https://github.com/zhuqi-lucas)) +- chore: extend record\_batch macro to support variables and expressions [\#9522](https://github.com/apache/arrow-rs/pull/9522) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([buraksenn](https://github.com/buraksenn)) +- \[Variant\] clean up `variant_get` tests [\#9518](https://github.com/apache/arrow-rs/pull/9518) ([sdf-jkl](https://github.com/sdf-jkl)) +- support large string for unshred variant [\#9515](https://github.com/apache/arrow-rs/pull/9515) ([friendlymatthew](https://github.com/friendlymatthew)) +- support string view unshred variant [\#9514](https://github.com/apache/arrow-rs/pull/9514) ([friendlymatthew](https://github.com/friendlymatthew)) +- Add has\_true\(\) and has\_false\(\) to BooleanArray [\#9511](https://github.com/apache/arrow-rs/pull/9511) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([adriangb](https://github.com/adriangb)) +- Fix Invalid offset in sparse column chunk data error for multiple predicates [\#9509](https://github.com/apache/arrow-rs/pull/9509) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([cetra3](https://github.com/cetra3)) +- fix: remove incorrect debug assertion in BatchCoalescer [\#9508](https://github.com/apache/arrow-rs/pull/9508) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Tim-53](https://github.com/Tim-53)) +- \[Json\] Add benchmarks for list json reader [\#9507](https://github.com/apache/arrow-rs/pull/9507) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- fix: first next\_back\(\) on new RowsIter panics [\#9505](https://github.com/apache/arrow-rs/pull/9505) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) +- Add some benchmarks for decoding delta encoded Parquet [\#9500](https://github.com/apache/arrow-rs/pull/9500) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://github.com/etseidl)) +- chore: remove duplicate macro `partially_shredded_variant_array_gen` [\#9498](https://github.com/apache/arrow-rs/pull/9498) ([codephage2020](https://github.com/codephage2020)) +- Deprecate ArrowTimestampType::make\_value in favor of from\_naive\_datetime [\#9491](https://github.com/apache/arrow-rs/pull/9491) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([codephage2020](https://github.com/codephage2020)) +- fix: Do not assume missing nullcount stat means zero nullcount [\#9481](https://github.com/apache/arrow-rs/pull/9481) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([scovich](https://github.com/scovich)) +- \[Variant\] Enahcne bracket access for VariantPath [\#9479](https://github.com/apache/arrow-rs/pull/9479) ([klion26](https://github.com/klion26)) +- Optimize delta binary decoder in the case where bitwidth=0 [\#9477](https://github.com/apache/arrow-rs/pull/9477) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://github.com/etseidl)) +- Add PrimitiveRunBuilder::with\_data\_type\(\) to customize the values' DataType [\#9473](https://github.com/apache/arrow-rs/pull/9473) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([brunal](https://github.com/brunal)) +- Convert `prettyprint` tests in `arrow-cast` to `insta` inline snapshots [\#9472](https://github.com/apache/arrow-rs/pull/9472) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([grtlr](https://github.com/grtlr)) +- Update strum\_macros requirement from 0.27 to 0.28 [\#9471](https://github.com/apache/arrow-rs/pull/9471) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([dependabot[bot]](https://github.com/apps/dependabot)) +- docs\(parquet\): Fix broken links in README [\#9467](https://github.com/apache/arrow-rs/pull/9467) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([SYaoJun](https://github.com/SYaoJun)) +- Add list-like types support to VariantArray::try\_new [\#9457](https://github.com/apache/arrow-rs/pull/9457) ([sdf-jkl](https://github.com/sdf-jkl)) +- Simplify downcast\_...!\(\) macro definitions [\#9454](https://github.com/apache/arrow-rs/pull/9454) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([brunal](https://github.com/brunal)) +- feat\(parquet\): add content defined chunking for arrow writer [\#9450](https://github.com/apache/arrow-rs/pull/9450) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([kszucs](https://github.com/kszucs)) +- refactor: simplify iterator using cloned\(\).map\(Some\) [\#9449](https://github.com/apache/arrow-rs/pull/9449) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([SYaoJun](https://github.com/SYaoJun)) +- feat: Optimize from\_bitwise\_binary\_op with 64-bit alignment [\#9441](https://github.com/apache/arrow-rs/pull/9441) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([kunalsinghdadhwal](https://github.com/kunalsinghdadhwal)) +- docs: fix markdown link syntax in README [\#9440](https://github.com/apache/arrow-rs/pull/9440) ([SYaoJun](https://github.com/SYaoJun)) +- Move `ListLikeArray` to arrow-array to be shared with json writer and parquet unshredding [\#9437](https://github.com/apache/arrow-rs/pull/9437) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- Add `claim` method to recordbatch for memory accounting [\#9433](https://github.com/apache/arrow-rs/pull/9433) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([cetra3](https://github.com/cetra3)) +- Add `append_nulls` to `MapBuilder` [\#9432](https://github.com/apache/arrow-rs/pull/9432) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Fokko](https://github.com/Fokko)) +- Add `append_non_nulls` to `StructBuilder` [\#9430](https://github.com/apache/arrow-rs/pull/9430) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Fokko](https://github.com/Fokko)) +- Add `append_value_n` to GenericByteBuilder [\#9426](https://github.com/apache/arrow-rs/pull/9426) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Fokko](https://github.com/Fokko)) +- refactor: simplify dynamic state for Avro record projection [\#9419](https://github.com/apache/arrow-rs/pull/9419) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mzabaluev](https://github.com/mzabaluev)) +- Add `NullBuffer::from_unsliced_buffer` helper and refactor call sites [\#9411](https://github.com/apache/arrow-rs/pull/9411) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Eyad3skr](https://github.com/Eyad3skr)) +- Implement min, max, sum for run-end-encoded arrays. [\#9409](https://github.com/apache/arrow-rs/pull/9409) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([brunal](https://github.com/brunal)) +- feat: add `RunArray::new_unchecked` and `RunArray::into_parts` [\#9376](https://github.com/apache/arrow-rs/pull/9376) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) +- Fix skip\_records over-counting when partial record precedes num\_rows page skip [\#9374](https://github.com/apache/arrow-rs/pull/9374) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([jonded94](https://github.com/jonded94)) +- fix: resolution of complex type variants in Avro unions [\#9328](https://github.com/apache/arrow-rs/pull/9328) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mzabaluev](https://github.com/mzabaluev)) +- feat\(arrow-avro\): Configurable Arrow timezone ID for Avro timestamps [\#9280](https://github.com/apache/arrow-rs/pull/9280) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mzabaluev](https://github.com/mzabaluev)) diff --git a/Cargo.toml b/Cargo.toml index 1a02830b0b9f..65043fb60ff4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,7 +68,7 @@ exclude = [ ] [workspace.package] -version = "58.0.0" +version = "58.1.0" homepage = "https://github.com/apache/arrow-rs" repository = "https://github.com/apache/arrow-rs" authors = ["Apache Arrow "] @@ -85,26 +85,26 @@ edition = "2024" rust-version = "1.85" [workspace.dependencies] -arrow = { version = "58.0.0", path = "./arrow", default-features = false } -arrow-arith = { version = "58.0.0", path = "./arrow-arith" } -arrow-array = { version = "58.0.0", path = "./arrow-array" } -arrow-buffer = { version = "58.0.0", path = "./arrow-buffer" } -arrow-cast = { version = "58.0.0", path = "./arrow-cast" } -arrow-csv = { version = "58.0.0", path = "./arrow-csv" } -arrow-data = { version = "58.0.0", path = "./arrow-data" } -arrow-ipc = { version = "58.0.0", path = "./arrow-ipc" } -arrow-json = { version = "58.0.0", path = "./arrow-json" } -arrow-ord = { version = "58.0.0", path = "./arrow-ord" } -arrow-pyarrow = { version = "58.0.0", path = "./arrow-pyarrow" } -arrow-row = { version = "58.0.0", path = "./arrow-row" } -arrow-schema = { version = "58.0.0", path = "./arrow-schema" } -arrow-select = { version = "58.0.0", path = "./arrow-select" } -arrow-string = { version = "58.0.0", path = "./arrow-string" } -parquet = { version = "58.0.0", path = "./parquet", default-features = false } -parquet-geospatial = { version = "58.0.0", path = "./parquet-geospatial" } -parquet-variant = { version = "58.0.0", path = "./parquet-variant" } -parquet-variant-json = { version = "58.0.0", path = "./parquet-variant-json" } -parquet-variant-compute = { version = "58.0.0", path = "./parquet-variant-compute" } +arrow = { version = "58.1.0", path = "./arrow", default-features = false } +arrow-arith = { version = "58.1.0", path = "./arrow-arith" } +arrow-array = { version = "58.1.0", path = "./arrow-array" } +arrow-buffer = { version = "58.1.0", path = "./arrow-buffer" } +arrow-cast = { version = "58.1.0", path = "./arrow-cast" } +arrow-csv = { version = "58.1.0", path = "./arrow-csv" } +arrow-data = { version = "58.1.0", path = "./arrow-data" } +arrow-ipc = { version = "58.1.0", path = "./arrow-ipc" } +arrow-json = { version = "58.1.0", path = "./arrow-json" } +arrow-ord = { version = "58.1.0", path = "./arrow-ord" } +arrow-pyarrow = { version = "58.1.0", path = "./arrow-pyarrow" } +arrow-row = { version = "58.1.0", path = "./arrow-row" } +arrow-schema = { version = "58.1.0", path = "./arrow-schema" } +arrow-select = { version = "58.1.0", path = "./arrow-select" } +arrow-string = { version = "58.1.0", path = "./arrow-string" } +parquet = { version = "58.1.0", path = "./parquet", default-features = false } +parquet-geospatial = { version = "58.1.0", path = "./parquet-geospatial" } +parquet-variant = { version = "58.1.0", path = "./parquet-variant" } +parquet-variant-json = { version = "58.1.0", path = "./parquet-variant-json" } +parquet-variant-compute = { version = "58.1.0", path = "./parquet-variant-compute" } chrono = { version = "0.4.40", default-features = false, features = ["clock"] } diff --git a/dev/release/update_change_log.sh b/dev/release/update_change_log.sh index b444cae64911..ae7bf81b0cca 100755 --- a/dev/release/update_change_log.sh +++ b/dev/release/update_change_log.sh @@ -29,8 +29,8 @@ set -e -SINCE_TAG="57.3.0" -FUTURE_RELEASE="58.0.0" +SINCE_TAG="58.0.0" +FUTURE_RELEASE="58.1.0" SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" From 6471e9ac72a79fd13963568ec3294a76fab826a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Fri, 20 Mar 2026 20:38:26 +0100 Subject: [PATCH 02/17] Pre-reserve output capacity in ByteView/ByteArray dictionary decoding (#9590) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Reserve `output.views` capacity in `ByteViewArrayDecoderDictionary::read` before the decode loop - Reserve `output.offsets` capacity in `ByteArrayDecoderDictionary::read` before the decode loop This avoids per-chunk reallocation during `extend` calls inside the dictionary decode loop. Closes #9587 ## Test plan - [ ] Existing tests pass (no functional change, only pre-allocation) - [ ] Benchmark dictionary-encoded StringView/BinaryView/String reads 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.6 (1M context) --- parquet/src/arrow/array_reader/byte_array.rs | 3 +++ parquet/src/arrow/array_reader/byte_view_array.rs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/parquet/src/arrow/array_reader/byte_array.rs b/parquet/src/arrow/array_reader/byte_array.rs index 0acbe6501924..2d0d44fbe203 100644 --- a/parquet/src/arrow/array_reader/byte_array.rs +++ b/parquet/src/arrow/array_reader/byte_array.rs @@ -580,6 +580,9 @@ impl ByteArrayDecoderDictionary { return Ok(0); } + // Pre-reserve offsets capacity to avoid per-chunk reallocation + output.offsets.reserve(len); + self.decoder.read(len, |keys| { output.extend_from_dictionary(keys, dict.offsets.as_slice(), dict.values.as_slice()) }) diff --git a/parquet/src/arrow/array_reader/byte_view_array.rs b/parquet/src/arrow/array_reader/byte_view_array.rs index 65b627aae451..1933654118f3 100644 --- a/parquet/src/arrow/array_reader/byte_view_array.rs +++ b/parquet/src/arrow/array_reader/byte_view_array.rs @@ -500,6 +500,9 @@ impl ByteViewArrayDecoderDictionary { // then the base_buffer_idx is 5 - 2 = 3 let base_buffer_idx = output.buffers.len() as u32 - dict.buffers.len() as u32; + // Pre-reserve output capacity to avoid per-chunk reallocation in extend + output.views.reserve(len); + let mut error = None; let read = self.decoder.read(len, |keys| { if base_buffer_idx == 0 { From 70445c54ff510e1b9f8d6e6782b92a1ce45fbb28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20G=C3=B6rtler?= Date: Sun, 22 Mar 2026 15:48:05 +0100 Subject: [PATCH 03/17] Add `quoted_strings` to `FormatOptions` (#9221) # Rationale for this change In some cases, it is desirable to print strings with surrounding quotation marks. A typical example that we run into in https://github.com/rerun-io/rerun is a `StructArray` that contains empty strings: Current formatting: ```text {name: } ``` Added option in this PR: ```text {name: ""} ``` # What changes are included in this PR? This PR relies on `std::fmt::Debug` to do the actual formatting of strings, which means that all escaping is handled out of the box. # Are these changes tested? This PR contains test for different types of inputs, including escape sequences. Additionally, it also tests the `StructArray` example outlined above. # Are there any user-facing changes? By default this option is false, making the feature opt-in. --------- Co-authored-by: Andrew Lamb --- arrow-cast/src/display.rs | 55 ++++++++++++-- arrow-cast/src/pretty.rs | 154 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 203 insertions(+), 6 deletions(-) diff --git a/arrow-cast/src/display.rs b/arrow-cast/src/display.rs index 59dfa26c93cb..0460c0c96b55 100644 --- a/arrow-cast/src/display.rs +++ b/arrow-cast/src/display.rs @@ -80,6 +80,8 @@ pub struct FormatOptions<'a> { duration_format: DurationFormat, /// Show types in visual representation batches types_info: bool, + /// Whether string values should be quoted + quoted_strings: bool, /// Formatter factory used to instantiate custom [`ArrayFormatter`]s. This allows users to /// provide custom formatters. formatter_factory: Option<&'a dyn ArrayFormatterFactory>, @@ -102,6 +104,7 @@ impl PartialEq for FormatOptions<'_> { && self.time_format == other.time_format && self.duration_format == other.duration_format && self.types_info == other.types_info + && self.quoted_strings == other.quoted_strings && match (self.formatter_factory, other.formatter_factory) { (Some(f1), Some(f2)) => std::ptr::eq(f1, f2), (None, None) => true, @@ -123,6 +126,7 @@ impl Hash for FormatOptions<'_> { self.time_format.hash(state); self.duration_format.hash(state); self.types_info.hash(state); + self.quoted_strings.hash(state); self.formatter_factory .map(|f| f as *const dyn ArrayFormatterFactory) .hash(state); @@ -142,6 +146,7 @@ impl<'a> FormatOptions<'a> { time_format: None, duration_format: DurationFormat::ISO8601, types_info: false, + quoted_strings: false, formatter_factory: None, } } @@ -217,6 +222,17 @@ impl<'a> FormatOptions<'a> { Self { types_info, ..self } } + /// Sets whether string values should be quoted + /// + /// When `true`, strings are formatted using [`Debug`]-style with double quotes and escaping. + /// Defaults to `false` + pub const fn with_quoted_strings(self, quoted_strings: bool) -> Self { + Self { + quoted_strings, + ..self + } + } + /// Overrides the [`ArrayFormatterFactory`] used to instantiate custom [`ArrayFormatter`]s. /// /// Using [`None`] causes pretty-printers to use the default [`ArrayFormatter`]s. @@ -276,6 +292,11 @@ impl<'a> FormatOptions<'a> { self.types_info } + /// Returns whether string values should be quoted. + pub const fn quoted_strings(&self) -> bool { + self.quoted_strings + } + /// Returns the [`ArrayFormatterFactory`] used to instantiate custom [`ArrayFormatter`]s. pub const fn formatter_factory(&self) -> Option<&'a dyn ArrayFormatterFactory> { self.formatter_factory @@ -1081,16 +1102,38 @@ impl Display for MillisecondsFormatter<'_> { } } -impl DisplayIndex for &GenericStringArray { - fn write(&self, idx: usize, f: &mut dyn Write) -> FormatResult { - write!(f, "{}", self.value(idx))?; +impl<'a, O: OffsetSizeTrait> DisplayIndexState<'a> for &'a GenericStringArray { + type State = bool; + + fn prepare(&self, options: &FormatOptions<'a>) -> Result { + Ok(options.quoted_strings()) + } + + fn write(&self, state: &Self::State, idx: usize, f: &mut dyn Write) -> FormatResult { + let value = self.value(idx); + if *state { + write!(f, "{:?}", value)?; + } else { + write!(f, "{}", value)?; + } Ok(()) } } -impl DisplayIndex for &StringViewArray { - fn write(&self, idx: usize, f: &mut dyn Write) -> FormatResult { - write!(f, "{}", self.value(idx))?; +impl<'a> DisplayIndexState<'a> for &'a StringViewArray { + type State = bool; + + fn prepare(&self, options: &FormatOptions<'a>) -> Result { + Ok(options.quoted_strings()) + } + + fn write(&self, state: &Self::State, idx: usize, f: &mut dyn Write) -> FormatResult { + let value = self.value(idx); + if *state { + write!(f, "{:?}", value)?; + } else { + write!(f, "{}", value)?; + } Ok(()) } } diff --git a/arrow-cast/src/pretty.rs b/arrow-cast/src/pretty.rs index e63147cd09c1..61ce5598992d 100644 --- a/arrow-cast/src/pretty.rs +++ b/arrow-cast/src/pretty.rs @@ -1665,4 +1665,158 @@ mod tests { .unwrap(); insta::assert_snapshot!(error, @"Invalid argument error: Expected the same number of columns in a record batch (1) as the number of fields (2) in the schema"); } + + #[test] + fn test_quoted_strings() { + let schema = Arc::new(Schema::new(vec![Field::new( + "strings", + DataType::Utf8, + true, + )])); + + let string_array = StringArray::from(vec![ + Some("hello"), + Some("world"), + Some(""), + Some("tab\there"), + Some("newline\ntest"), + Some("quote\"test"), + Some("backslash\\test"), + None, + ]); + + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(string_array)]).unwrap(); + + let options_none = FormatOptions::new().with_null("NULL"); + let table = pretty_format_batches_with_options(std::slice::from_ref(&batch), &options_none) + .unwrap() + .to_string(); + + insta::assert_snapshot!(table, @" + +----------------+ + | strings | + +----------------+ + | hello | + | world | + | | + | tab here | + | newline | + | test | + | quote\"test | + | backslash\\test | + | NULL | + +----------------+ + "); + + let options_quoted = FormatOptions::new() + .with_null("NULL") + .with_quoted_strings(true); + + let table = pretty_format_batches_with_options(&[batch], &options_quoted) + .unwrap() + .to_string(); + + insta::assert_snapshot!(table, @r#" + +-------------------+ + | strings | + +-------------------+ + | "hello" | + | "world" | + | "" | + | "tab\there" | + | "newline\ntest" | + | "quote\"test" | + | "backslash\\test" | + | NULL | + +-------------------+ + "#); + } + + #[test] + fn test_string_view_quoted() { + let schema = Arc::new(Schema::new(vec![Field::new( + "view_strings", + DataType::Utf8View, + true, + )])); + + let mut builder = StringViewBuilder::new(); + builder.append_value("hello"); + builder.append_null(); + builder.append_value("quote\"test"); + + let array: ArrayRef = Arc::new(builder.finish()); + let batch = RecordBatch::try_new(schema, vec![array]).unwrap(); + + let options = FormatOptions::new().with_quoted_strings(true); + + let table = pretty_format_batches_with_options(&[batch], &options) + .unwrap() + .to_string(); + + insta::assert_snapshot!(table, @" + +---------------+ + | view_strings | + +---------------+ + | \"hello\" | + | | + | \"quote\\\"test\" | + +---------------+ + "); + } + + #[test] + fn test_quoted_strings_in_struct() { + let string_builder = StringBuilder::new(); + let mut name_builder = string_builder; + name_builder.append_value("Alice"); + name_builder.append_value(""); + name_builder.append_value("Bob"); + + let fields = vec![Field::new("name", DataType::Utf8, false)]; + let mut struct_builder = StructBuilder::new(fields, vec![Box::new(name_builder)]); + struct_builder.append(true); + struct_builder.append(true); + struct_builder.append(true); + + let struct_array = struct_builder.finish(); + + let schema = Arc::new(Schema::new(vec![Field::new( + "person", + struct_array.data_type().clone(), + false, + )])); + + let batch = RecordBatch::try_new(schema, vec![Arc::new(struct_array)]).unwrap(); + + let options_none = FormatOptions::new(); + let table = pretty_format_batches_with_options(std::slice::from_ref(&batch), &options_none) + .unwrap() + .to_string(); + + insta::assert_snapshot!(table, @" + +---------------+ + | person | + +---------------+ + | {name: Alice} | + | {name: } | + | {name: Bob} | + +---------------+ + "); + + let options_quoted = FormatOptions::new().with_quoted_strings(true); + let table = pretty_format_batches_with_options(&[batch], &options_quoted) + .unwrap() + .to_string(); + + insta::assert_snapshot!(table, @" + +-----------------+ + | person | + +-----------------+ + | {name: \"Alice\"} | + | {name: \"\"} | + | {name: \"Bob\"} | + +-----------------+ + "); + } } From 980ea0b36c79a9e996efd90ad5f24571f0f9c0e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Tue, 24 Mar 2026 14:03:23 +0100 Subject: [PATCH 04/17] Reduce per-byte overhead in VLQ integer decoding (#9584) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Which issue does this PR close? Closes #9580 ## Rationale The current VLQ decoder calls `get_aligned` for each byte, which involves repeated offset calculations and bounds checks in the hot loop. ## What changes are included in this PR? Align to the byte boundary once, then iterate directly over the buffer slice, avoiding per-byte overhead from `get_aligned`. ## Are there any user-facing changes? No. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.6 (1M context) --- parquet/src/util/bit_util.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/parquet/src/util/bit_util.rs b/parquet/src/util/bit_util.rs index 3a26603fabc4..262564825864 100644 --- a/parquet/src/util/bit_util.rs +++ b/parquet/src/util/bit_util.rs @@ -659,9 +659,15 @@ impl BitReader { /// /// Returns `None` if there's not enough bytes in the stream. `Some` otherwise. pub fn get_vlq_int(&mut self) -> Option { + // Align to byte boundary once, then read bytes directly + self.byte_offset = self.get_byte_offset(); + self.bit_offset = 0; + + let buf = &self.buffer[self.byte_offset..]; let mut shift = 0; let mut v: i64 = 0; - while let Some(byte) = self.get_aligned::(1) { + + for (i, &byte) in buf.iter().enumerate() { v |= ((byte & 0x7F) as i64) << shift; shift += 7; assert!( @@ -669,6 +675,7 @@ impl BitReader { "Num of bytes exceed MAX_VLQ_BYTE_LEN ({MAX_VLQ_BYTE_LEN})" ); if byte & 0x80 == 0 { + self.byte_offset += i + 1; return Some(v); } } From 398962ec67bc777eca1c635c8ef01a9c634530eb Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Wed, 25 Mar 2026 20:05:53 +0200 Subject: [PATCH 05/17] deps: fix `object_store` breakage for 0.13.2 (#9612) # Rationale for this change The `object_store` crate release 0.13.2 breaks the build of parquet because it feature-gates the `buffered` module. I have filed https://github.com/apache/arrow-rs-object-store/issues/677 about the breakage; meanwhile this fix is made in expectation that 0.13.2 will not be yanked and the feature gate will remain. # What changes are included in this PR? Bump the version to 0.13.2 and requesting the "tokio" feature. # Are these changes tested? The build should succeed in CI workflows. # Are there any user-facing changes? No Co-authored-by: Mikhail Zabaluev --- Cargo.toml | 6 ++++-- arrow-avro/Cargo.toml | 4 ++-- parquet/Cargo.toml | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 65043fb60ff4..4ca4b068f65a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,12 +108,14 @@ parquet-variant-compute = { version = "58.1.0", path = "./parquet-variant-comput chrono = { version = "0.4.40", default-features = false, features = ["clock"] } -simdutf8 = { version = "0.1.5", default-features = false } - criterion = { version = "0.8.0", default-features = false } insta = { version = "1.46.3", default-features = false } +object_store = { version = "0.13.2", default-features = false } + +simdutf8 = { version = "0.1.5", default-features = false } + # release inherited profile keeping debug information and symbols # for mem/cpu profiling [profile.profiling] diff --git a/arrow-avro/Cargo.toml b/arrow-avro/Cargo.toml index 93eb825f9a7b..7f17c98431ea 100644 --- a/arrow-avro/Cargo.toml +++ b/arrow-avro/Cargo.toml @@ -56,7 +56,7 @@ arrow-buffer = { workspace = true } arrow-array = { workspace = true } arrow-select = { workspace = true, optional = true } -object_store = { version = "0.13", default-features = false, optional = true } +object_store = { workspace = true, optional = true } bytes = { version = "1.11.0", default-features = false, features = ["std"] } serde_json = { version = "1.0", default-features = false, features = ["std"] } @@ -93,7 +93,7 @@ futures = "0.3.31" async-stream = "0.3.6" apache-avro = "0.21.0" num-bigint = "0.4" -object_store = { version = "0.13", default-features = false, features = ["fs"] } +object_store = { workspace = true, features = ["fs"] } once_cell = "1.21.3" half = { version = "2.1", default-features = false } tokio = { version = "1.0", default-features = false, features = ["macros", "rt-multi-thread", "io-util", "fs"] } diff --git a/parquet/Cargo.toml b/parquet/Cargo.toml index 4be779302448..efcd1fe2190b 100644 --- a/parquet/Cargo.toml +++ b/parquet/Cargo.toml @@ -49,7 +49,7 @@ parquet-variant = { workspace = true, optional = true } parquet-variant-json = { workspace = true, optional = true } parquet-variant-compute = { workspace = true, optional = true } -object_store = { version = "0.13.1", default-features = false, optional = true } +object_store = { workspace = true, optional = true, features = ["tokio"] } bytes = { version = "1.1", default-features = false, features = ["std"] } thrift = { version = "0.17", default-features = false } @@ -93,7 +93,7 @@ arrow = { workspace = true, features = ["ipc", "test_utils", "prettyprint", "jso arrow-cast = { workspace = true } tokio = { version = "1.0", default-features = false, features = ["macros", "rt-multi-thread", "io-util", "fs"] } rand = { version = "0.9", default-features = false, features = ["std", "std_rng", "thread_rng"] } -object_store = { version = "0.13.1", default-features = false, features = ["azure", "fs"] } +object_store = { workspace = true, features = ["azure", "fs"] } sysinfo = { version = "0.38.1", default-features = false, features = ["system"] } [package.metadata.docs.rs] From f2512b5341ec66dcafe9de94ae382401ce5e8698 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 18:11:17 -0400 Subject: [PATCH 06/17] chore(deps): update sha2 requirement from 0.10 to 0.11 (#9618) Updates the requirements on [sha2](https://github.com/RustCrypto/hashes) to permit the latest version.
Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- arrow-avro/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arrow-avro/Cargo.toml b/arrow-avro/Cargo.toml index 7f17c98431ea..f46ef7e7b999 100644 --- a/arrow-avro/Cargo.toml +++ b/arrow-avro/Cargo.toml @@ -75,7 +75,7 @@ uuid = "1.17" indexmap = "2.10" rand = "0.9" md5 = { version = "0.8", optional = true } -sha2 = { version = "0.10", optional = true } +sha2 = { version = "0.11", optional = true } tokio = { version = "1.0", optional = true, default-features = false, features = ["macros", "rt", "io-util"] } [dev-dependencies] From 1f1c3a4cea6972ade7ff73a7765521c21a992e4f Mon Sep 17 00:00:00 2001 From: Liam Bao Date: Thu, 26 Mar 2026 18:15:41 -0400 Subject: [PATCH 07/17] Support `ListView` codec in arrow-json (#9503) # Which issue does this PR close? - Closes #9340. # Rationale for this change # What changes are included in this PR? Support `ListView` codec in arrow-json. Using `ListLikeArray` trait to simplify implementation. # Are these changes tested? Tests added # Are there any user-facing changes? New encoder/decoder --- arrow-json/src/lib.rs | 66 +++++++++++++++++++----- arrow-json/src/reader/list_array.rs | 43 +++++++++++----- arrow-json/src/reader/mod.rs | 80 ++++++++++++++++++++++++++++- arrow-json/src/writer/encoder.rs | 79 ++++++++-------------------- arrow-json/src/writer/mod.rs | 48 +++++++++++++++++ 5 files changed, 228 insertions(+), 88 deletions(-) diff --git a/arrow-json/src/lib.rs b/arrow-json/src/lib.rs index 1b18e0094708..201c3cd80749 100644 --- a/arrow-json/src/lib.rs +++ b/arrow-json/src/lib.rs @@ -179,17 +179,17 @@ impl JsonSerializable for f64 { #[cfg(test)] mod tests { - use std::sync::Arc; - - use crate::writer::JsonArray; - use super::*; - + use crate::writer::JsonArray; + use crate::writer::LineDelimited; use arrow_array::{ - ArrayRef, GenericBinaryArray, GenericByteViewArray, RecordBatch, RecordBatchWriter, - builder::FixedSizeBinaryBuilder, types::BinaryViewType, + ArrayRef, GenericBinaryArray, GenericByteViewArray, GenericListViewArray, RecordBatch, + RecordBatchWriter, builder::FixedSizeBinaryBuilder, types::BinaryViewType, }; + use arrow_schema::{DataType, Field, Fields, Schema}; use serde_json::Value::{Bool, Number as VNumber, String as VString}; + use std::io::Cursor; + use std::sync::Arc; #[test] fn test_arrow_native_type_to_json() { @@ -216,13 +216,6 @@ mod tests { #[test] fn test_json_roundtrip_structs() { - use crate::writer::LineDelimited; - use arrow_schema::DataType; - use arrow_schema::Field; - use arrow_schema::Fields; - use arrow_schema::Schema; - use std::sync::Arc; - let schema = Arc::new(Schema::new(vec![ Field::new( "c1", @@ -352,4 +345,49 @@ mod tests { assert_eq!(batch, decoded); } + + fn assert_list_view_roundtrip() { + let flat_field = Arc::new(Field::new("item", DataType::Int32, true)); + let flat_dt = GenericListViewArray::::DATA_TYPE_CONSTRUCTOR(flat_field); + + let nested_inner = Arc::new(Field::new("item", DataType::Int32, false)); + let nested_inner_dt = GenericListViewArray::::DATA_TYPE_CONSTRUCTOR(nested_inner); + let nested_outer = Arc::new(Field::new("item", nested_inner_dt, true)); + let nested_dt = GenericListViewArray::::DATA_TYPE_CONSTRUCTOR(nested_outer); + + let schema = Arc::new(Schema::new(vec![ + Field::new("flat", flat_dt, true), + Field::new("nested", nested_dt, true), + ])); + + let input = r#"{"flat":[1,2,3],"nested":[[1,2],[3]]} +{"flat":[4,null]} +{} +{"flat":[6],"nested":[[4,5,6]]} +{"flat":[]} +"# + .as_bytes(); + + let batches: Vec = ReaderBuilder::new(schema.clone()) + .with_batch_size(1024) + .build(Cursor::new(input)) + .unwrap() + .collect::, _>>() + .unwrap(); + + let mut output = Vec::new(); + let mut writer = WriterBuilder::new().build::<_, LineDelimited>(&mut output); + for batch in &batches { + writer.write(batch).unwrap(); + } + writer.finish().unwrap(); + + assert_eq!(input, &output); + } + + #[test] + fn test_json_roundtrip_list_view() { + assert_list_view_roundtrip::(); + assert_list_view_roundtrip::(); + } } diff --git a/arrow-json/src/reader/list_array.rs b/arrow-json/src/reader/list_array.rs index d363b6be9780..ea23403c4b18 100644 --- a/arrow-json/src/reader/list_array.rs +++ b/arrow-json/src/reader/list_array.rs @@ -18,28 +18,33 @@ use crate::reader::tape::{Tape, TapeElement}; use crate::reader::{ArrayDecoder, DecoderContext}; use arrow_array::OffsetSizeTrait; -use arrow_array::builder::{BooleanBufferBuilder, BufferBuilder}; -use arrow_buffer::buffer::NullBuffer; +use arrow_array::builder::BooleanBufferBuilder; +use arrow_buffer::{Buffer, buffer::NullBuffer}; use arrow_data::{ArrayData, ArrayDataBuilder}; use arrow_schema::{ArrowError, DataType}; use std::marker::PhantomData; -pub struct ListArrayDecoder { +pub type ListArrayDecoder = ListLikeArrayDecoder; +pub type ListViewArrayDecoder = ListLikeArrayDecoder; + +pub struct ListLikeArrayDecoder { data_type: DataType, decoder: Box, phantom: PhantomData, is_nullable: bool, } -impl ListArrayDecoder { +impl ListLikeArrayDecoder { pub fn new( ctx: &DecoderContext, data_type: &DataType, is_nullable: bool, ) -> Result { - let field = match data_type { - DataType::List(f) if !O::IS_LARGE => f, - DataType::LargeList(f) if O::IS_LARGE => f, + let field = match (IS_VIEW, data_type) { + (false, DataType::List(f)) if !O::IS_LARGE => f, + (false, DataType::LargeList(f)) if O::IS_LARGE => f, + (true, DataType::ListView(f)) if !O::IS_LARGE => f, + (true, DataType::LargeListView(f)) if O::IS_LARGE => f, _ => unreachable!(), }; let decoder = ctx.make_decoder(field.data_type(), field.is_nullable())?; @@ -53,11 +58,11 @@ impl ListArrayDecoder { } } -impl ArrayDecoder for ListArrayDecoder { +impl ArrayDecoder for ListLikeArrayDecoder { fn decode(&mut self, tape: &Tape<'_>, pos: &[u32]) -> Result { let mut child_pos = Vec::with_capacity(pos.len()); - let mut offsets = BufferBuilder::::new(pos.len() + 1); - offsets.append(O::from_usize(0).unwrap()); + let mut offsets = Vec::with_capacity(pos.len() + 1); + offsets.push(O::from_usize(0).unwrap()); let mut nulls = self .is_nullable @@ -88,18 +93,30 @@ impl ArrayDecoder for ListArrayDecoder { let offset = O::from_usize(child_pos.len()).ok_or_else(|| { ArrowError::JsonError(format!("offset overflow decoding {}", self.data_type)) })?; - offsets.append(offset) + offsets.push(offset); } let child_data = self.decoder.decode(tape, &child_pos)?; let nulls = nulls.as_mut().map(|x| NullBuffer::new(x.finish())); - let data = ArrayDataBuilder::new(self.data_type.clone()) + let mut data = ArrayDataBuilder::new(self.data_type.clone()) .len(pos.len()) .nulls(nulls) - .add_buffer(offsets.finish()) .child_data(vec![child_data]); + if IS_VIEW { + let mut sizes = Vec::with_capacity(offsets.len() - 1); + for i in 1..offsets.len() { + sizes.push(offsets[i] - offsets[i - 1]); + } + offsets.pop(); + data = data + .add_buffer(Buffer::from_vec(offsets)) + .add_buffer(Buffer::from_vec(sizes)); + } else { + data = data.add_buffer(Buffer::from_vec(offsets)); + } + // Safety // Validated lengths above Ok(unsafe { data.build_unchecked() }) diff --git a/arrow-json/src/reader/mod.rs b/arrow-json/src/reader/mod.rs index 04271368a4aa..7039d3500ece 100644 --- a/arrow-json/src/reader/mod.rs +++ b/arrow-json/src/reader/mod.rs @@ -154,7 +154,7 @@ pub use value_iter::ValueIter; use crate::reader::boolean_array::BooleanArrayDecoder; use crate::reader::decimal_array::DecimalArrayDecoder; -use crate::reader::list_array::ListArrayDecoder; +use crate::reader::list_array::{ListArrayDecoder, ListViewArrayDecoder}; use crate::reader::map_array::MapArrayDecoder; use crate::reader::null_array::NullArrayDecoder; use crate::reader::primitive_array::PrimitiveArrayDecoder; @@ -792,6 +792,8 @@ fn make_decoder( DataType::LargeUtf8 => Ok(Box::new(StringArrayDecoder::::new(coerce_primitive))), DataType::List(_) => Ok(Box::new(ListArrayDecoder::::new(ctx, data_type, is_nullable)?)), DataType::LargeList(_) => Ok(Box::new(ListArrayDecoder::::new(ctx, data_type, is_nullable)?)), + DataType::ListView(_) => Ok(Box::new(ListViewArrayDecoder::::new(ctx, data_type, is_nullable)?)), + DataType::LargeListView(_) => Ok(Box::new(ListViewArrayDecoder::::new(ctx, data_type, is_nullable)?)), DataType::Struct(_) => Ok(Box::new(StructArrayDecoder::new(ctx, data_type, is_nullable)?)), DataType::Binary => Ok(Box::new(BinaryArrayDecoder::::default())), DataType::LargeBinary => Ok(Box::new(BinaryArrayDecoder::::default())), @@ -815,7 +817,10 @@ mod tests { use std::io::{BufReader, Cursor, Seek}; use arrow_array::cast::AsArray; - use arrow_array::{Array, BooleanArray, Float64Array, ListArray, StringArray, StringViewArray}; + use arrow_array::{ + Array, BooleanArray, Float64Array, GenericListViewArray, ListArray, OffsetSizeTrait, + StringArray, StringViewArray, + }; use arrow_buffer::{ArrowNativeType, Buffer}; use arrow_cast::display::{ArrayFormatter, FormatOptions}; use arrow_data::ArrayDataBuilder; @@ -2192,6 +2197,77 @@ mod tests { assert_eq!(read, expected); } + fn assert_read_list_view() { + let field = Arc::new(Field::new("item", DataType::Int32, true)); + let data_type = GenericListViewArray::::DATA_TYPE_CONSTRUCTOR(field.clone()); + let schema = Arc::new(Schema::new(vec![Field::new("lv", data_type, true)])); + + let buf = r#" + {"lv": [1, 2, 3]} + {"lv": [4, null]} + {"lv": null} + {"lv": [6]} + {"lv": []} + "#; + + let batches = do_read(buf, 1024, false, false, schema); + assert_eq!(batches.len(), 1); + let batch = &batches[0]; + let col = batch.column(0); + let list_view = col + .as_any() + .downcast_ref::>() + .unwrap(); + + assert_eq!(list_view.len(), 5); + + // Check offsets and sizes + let expected_offsets: Vec = vec![0, 3, 5, 5, 6] + .into_iter() + .map(|v| O::usize_as(v)) + .collect(); + let expected_sizes: Vec = vec![3, 2, 0, 1, 0] + .into_iter() + .map(|v| O::usize_as(v)) + .collect(); + assert_eq!(list_view.value_offsets(), &expected_offsets); + assert_eq!(list_view.value_sizes(), &expected_sizes); + + // Row 0: [1, 2, 3] + assert!(list_view.is_valid(0)); + let vals = list_view.value(0); + let ints = vals.as_primitive::(); + assert_eq!(ints.values(), &[1, 2, 3]); + + // Row 1: [4, null] + assert!(list_view.is_valid(1)); + let vals = list_view.value(1); + let ints = vals.as_primitive::(); + assert_eq!(ints.len(), 2); + assert_eq!(ints.value(0), 4); + assert!(ints.is_null(1)); + + // Row 2: null + assert!(list_view.is_null(2)); + + // Row 3: [6] + assert!(list_view.is_valid(3)); + let vals = list_view.value(3); + let ints = vals.as_primitive::(); + assert_eq!(ints.values(), &[6]); + + // Row 4: [] + assert!(list_view.is_valid(4)); + let vals = list_view.value(4); + assert_eq!(vals.len(), 0); + } + + #[test] + fn test_read_list_view() { + assert_read_list_view::(); + assert_read_list_view::(); + } + #[test] fn test_skip_empty_lines() { let schema = Schema::new(vec![Field::new("a", DataType::Int64, true)]); diff --git a/arrow-json/src/writer/encoder.rs b/arrow-json/src/writer/encoder.rs index d7c3fbbe2e34..45055c5a36a8 100644 --- a/arrow-json/src/writer/encoder.rs +++ b/arrow-json/src/writer/encoder.rs @@ -352,15 +352,23 @@ pub fn make_encoder<'a>( } DataType::List(_) => { let array = array.as_list::(); - NullableEncoder::new(Box::new(ListEncoder::try_new(field, array, options)?), array.nulls().cloned()) + NullableEncoder::new(Box::new(ListLikeEncoder::try_new(field, array, options)?), array.nulls().cloned()) } DataType::LargeList(_) => { let array = array.as_list::(); - NullableEncoder::new(Box::new(ListEncoder::try_new(field, array, options)?), array.nulls().cloned()) + NullableEncoder::new(Box::new(ListLikeEncoder::try_new(field, array, options)?), array.nulls().cloned()) + } + DataType::ListView(_) => { + let array = array.as_list_view::(); + NullableEncoder::new(Box::new(ListLikeEncoder::try_new(field, array, options)?), array.nulls().cloned()) + } + DataType::LargeListView(_) => { + let array = array.as_list_view::(); + NullableEncoder::new(Box::new(ListLikeEncoder::try_new(field, array, options)?), array.nulls().cloned()) } DataType::FixedSizeList(_, _) => { let array = array.as_fixed_size_list(); - NullableEncoder::new(Box::new(FixedSizeListEncoder::try_new(field, array, options)?), array.nulls().cloned()) + NullableEncoder::new(Box::new(ListLikeEncoder::try_new(field, array, options)?), array.nulls().cloned()) } DataType::Dictionary(_, _) => downcast_dictionary_array! { @@ -639,77 +647,30 @@ impl Encoder for BinaryViewEncoder<'_> { } } -struct ListEncoder<'a, O: OffsetSizeTrait> { - offsets: OffsetBuffer, - encoder: NullableEncoder<'a>, -} - -impl<'a, O: OffsetSizeTrait> ListEncoder<'a, O> { - fn try_new( - field: &'a FieldRef, - array: &'a GenericListArray, - options: &'a EncoderOptions, - ) -> Result { - let encoder = make_encoder(field, array.values().as_ref(), options)?; - Ok(Self { - offsets: array.offsets().clone(), - encoder, - }) - } -} - -impl Encoder for ListEncoder<'_, O> { - fn encode(&mut self, idx: usize, out: &mut Vec) { - let end = self.offsets[idx + 1].as_usize(); - let start = self.offsets[idx].as_usize(); - out.push(b'['); - - if self.encoder.has_nulls() { - for idx in start..end { - if idx != start { - out.push(b',') - } - if self.encoder.is_null(idx) { - out.extend_from_slice(b"null"); - } else { - self.encoder.encode(idx, out); - } - } - } else { - for idx in start..end { - if idx != start { - out.push(b',') - } - self.encoder.encode(idx, out); - } - } - out.push(b']'); - } -} - -struct FixedSizeListEncoder<'a> { - value_length: usize, +struct ListLikeEncoder<'a, L: ListLikeArray> { + list_array: &'a L, encoder: NullableEncoder<'a>, } -impl<'a> FixedSizeListEncoder<'a> { +impl<'a, L: ListLikeArray> ListLikeEncoder<'a, L> { fn try_new( field: &'a FieldRef, - array: &'a FixedSizeListArray, + array: &'a L, options: &'a EncoderOptions, ) -> Result { let encoder = make_encoder(field, array.values().as_ref(), options)?; Ok(Self { + list_array: array, encoder, - value_length: array.value_length().as_usize(), }) } } -impl Encoder for FixedSizeListEncoder<'_> { +impl Encoder for ListLikeEncoder<'_, L> { fn encode(&mut self, idx: usize, out: &mut Vec) { - let start = idx * self.value_length; - let end = start + self.value_length; + let range = self.list_array.element_range(idx); + let start = range.start; + let end = range.end; out.push(b'['); if self.encoder.has_nulls() { for idx in start..end { diff --git a/arrow-json/src/writer/mod.rs b/arrow-json/src/writer/mod.rs index 2fac5ab62353..04cc8c9e2a93 100644 --- a/arrow-json/src/writer/mod.rs +++ b/arrow-json/src/writer/mod.rs @@ -1241,6 +1241,54 @@ mod tests { ); } + fn assert_write_list_view() { + let field = Arc::new(Field::new("item", DataType::Int32, true)); + let data_type = GenericListViewArray::::DATA_TYPE_CONSTRUCTOR(field.clone()); + let schema = Schema::new(vec![Field::new("lv", data_type, true)]); + + // rows: [1, 2, 3], [4, null], null, [6] + let values = Int32Array::from(vec![Some(1), Some(2), Some(3), Some(4), None, Some(6)]); + let offsets = [0, 3, 0, 5] + .iter() + .map(|&v| O::from_usize(v).unwrap()) + .collect::>(); + let sizes = [3, 2, 0, 1] + .iter() + .map(|&v| O::from_usize(v).unwrap()) + .collect::>(); + let list_view = GenericListViewArray::::try_new( + field, + ScalarBuffer::from(offsets), + ScalarBuffer::from(sizes), + Arc::new(values), + Some(NullBuffer::from_iter([true, true, false, true])), + ) + .unwrap(); + + let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(list_view)]).unwrap(); + + let mut buf = Vec::new(); + { + let mut writer = LineDelimitedWriter::new(&mut buf); + writer.write_batches(&[&batch]).unwrap(); + } + + assert_json_eq( + &buf, + r#"{"lv":[1,2,3]} +{"lv":[4,null]} +{} +{"lv":[6]} +"#, + ); + } + + #[test] + fn write_list_view() { + assert_write_list_view::(); + assert_write_list_view::(); + } + fn test_write_for_file(test_file: &str, remove_nulls: bool) { let file = File::open(test_file).unwrap(); let mut reader = BufReader::new(file); From 7f307c031f31a691be566f5e20171455c41dd661 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 26 Mar 2026 23:17:50 +0100 Subject: [PATCH 08/17] fix: Stop using https://dist.apache.org/repos/dist/dev/arrow/KEYS for verification (#9604) # Which issue does this PR close? - Closes #9603 # Rationale for this change The release and dev KEYS files could get out of synch. We should use the release/ version: - Users use the release/ version not dev/ version when they verify our artifacts' signature - https://dist.apache.org/ may reject our request when we request many times by CI # What changes are included in this PR? Use `https://www.apache.org/dyn/closer.lua?action=download&filename=arrow/KEYS` to download the KEYS file and the expected `https://dist.apache.org/repos/dist/dev/arrow` for the RC artifacts. # Are these changes tested? Yes, I've verified 58.1.0 1 both previous to the change and after the change. # Are there any user-facing changes? No --- dev/release/verify-release-candidate.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 2629d362aaff..d8b888effef1 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -33,23 +33,27 @@ set -o pipefail SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" ARROW_DIR="$(dirname $(dirname ${SOURCE_DIR}))" -ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow' +ARROW_RC_URL="https://dist.apache.org/repos/dist/dev/arrow" +ARROW_KEYS_URL="https://www.apache.org/dyn/closer.lua?action=download&filename=arrow/KEYS" -download_dist_file() { +download_file() { curl \ --silent \ --show-error \ --fail \ --location \ - --remote-name $ARROW_DIST_URL/$1 + --output "$2" \ + "$1" } download_rc_file() { - download_dist_file apache-arrow-rs-${VERSION}-rc${RC_NUMBER}/$1 + download_file \ + "${ARROW_RC_URL}/apache-arrow-rs-${VERSION}-rc${RC_NUMBER}/$1" \ + "$1" } import_gpg_keys() { - download_dist_file KEYS + download_file "${ARROW_KEYS_URL}" KEYS gpg --import KEYS } From c194e54dc4e8fcd8b9333eea2528d5db1c1ba912 Mon Sep 17 00:00:00 2001 From: Konstantin Tarasov <33369833+sdf-jkl@users.noreply.github.com> Date: Thu, 26 Mar 2026 18:18:42 -0400 Subject: [PATCH 09/17] [Variant] Add unshredded `Struct` fast-path for `variant_get(..., Struct)` (#9597) # Which issue does this PR close? - Closes #9596. # Rationale for this change Check issue # What changes are included in this PR? Reuse `shred_basic_variant` as a fast path for unshredded `Struct` handling in `variant_get(..., Struct)` # Are these changes tested? Yes, added two unit tests to establish safe mode behavior. # Are there any user-facing changes? --- parquet-variant-compute/src/variant_get.rs | 90 +++++++++++++++++++++- 1 file changed, 88 insertions(+), 2 deletions(-) diff --git a/parquet-variant-compute/src/variant_get.rs b/parquet-variant-compute/src/variant_get.rs index 9204dcf70856..3e9892cacf70 100644 --- a/parquet-variant-compute/src/variant_get.rs +++ b/parquet-variant-compute/src/variant_get.rs @@ -213,9 +213,20 @@ fn shredded_get_path( return Ok(shredded); } - // Structs are special. Recurse into each field separately, hoping to follow the shredding even - // further, and build up the final struct from those individually shredded results. + // Structs are special. + // + // For fully unshredded targets (`typed_value` absent), delegate to the row builder so we + // preserve struct-level cast semantics: + // - safe mode: non-object rows become NULL structs + // - strict mode: non-object rows raise a cast error + // + // For shredded/partially-shredded targets (`typed_value` present), recurse into each field + // separately to take advantage of deeper shredding in child fields. if let DataType::Struct(fields) = as_field.data_type() { + if target.typed_value_field().is_none() { + return shred_basic_variant(target, VariantPath::default(), Some(as_field)); + } + let children = fields .iter() .map(|field| { @@ -3111,6 +3122,81 @@ mod test { assert_eq!(inner_values.value(1), 100); } + #[test] + fn test_unshredded_struct_safe_cast_non_object_rows_are_null() { + let json_strings = vec![r#"{"a": 1, "b": 2}"#, "123", "{}"]; + let string_array: Arc = Arc::new(StringArray::from(json_strings)); + let variant_array_ref = ArrayRef::from(json_to_variant(&string_array).unwrap()); + + let struct_fields = Fields::from(vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Int32, true), + ]); + let options = GetOptions { + path: VariantPath::default(), + as_type: Some(Arc::new(Field::new( + "result", + DataType::Struct(struct_fields), + true, + ))), + cast_options: CastOptions::default(), + }; + + let result = variant_get(&variant_array_ref, options).unwrap(); + let struct_result = result.as_struct(); + let field_a = struct_result + .column(0) + .as_primitive::(); + let field_b = struct_result + .column(1) + .as_primitive::(); + + // Row 0 is an object, so the struct row is valid with extracted fields. + assert!(!struct_result.is_null(0)); + assert_eq!(field_a.value(0), 1); + assert_eq!(field_b.value(0), 2); + + // Row 1 is a scalar, so safe struct cast should produce a NULL struct row. + assert!(struct_result.is_null(1)); + assert!(field_a.is_null(1)); + assert!(field_b.is_null(1)); + + // Row 2 is an empty object, so the struct row is valid with missing fields as NULL. + assert!(!struct_result.is_null(2)); + assert!(field_a.is_null(2)); + assert!(field_b.is_null(2)); + } + + #[test] + fn test_unshredded_struct_strict_cast_non_object_errors() { + let json_strings = vec![r#"{"a": 1, "b": 2}"#, "123"]; + let string_array: Arc = Arc::new(StringArray::from(json_strings)); + let variant_array_ref = ArrayRef::from(json_to_variant(&string_array).unwrap()); + + let struct_fields = Fields::from(vec![ + Field::new("a", DataType::Int32, true), + Field::new("b", DataType::Int32, true), + ]); + let options = GetOptions { + path: VariantPath::default(), + as_type: Some(Arc::new(Field::new( + "result", + DataType::Struct(struct_fields), + true, + ))), + cast_options: CastOptions { + safe: false, + ..Default::default() + }, + }; + + let err = variant_get(&variant_array_ref, options).unwrap_err(); + assert!( + err.to_string() + .contains("Failed to extract struct from variant") + ); + } + /// Create comprehensive shredded variant with diverse null patterns and empty objects /// Rows: normal values, top-level null, missing field a, missing field b, empty object fn create_comprehensive_shredded_variant() -> ArrayRef { From aa9432c8833f5701085e8b933b30560d21df9f80 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Sun, 29 Mar 2026 23:11:05 -0700 Subject: [PATCH 10/17] Fix `extend_nulls` panic for UnionArray (#9607) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Fix `MutableArrayData::extend_nulls` which previously panicked unconditionally for both sparse and dense Union arrays - For sparse unions: append the first type_id and extend nulls in all children - For dense unions: append the first type_id, compute offsets into the first child, and extend nulls in that child only ## Background This bug was discovered via DataFusion. `CaseExpr` uses `MutableArrayData` via `scatter()` to build result arrays. When a `CASE` expression returns a Union type (e.g., from `json_get` which returns a JSON union) and there are rows where no `WHEN` branch matches (implicit `ELSE NULL`), `scatter` calls `extend_nulls` which panics with "cannot call extend_nulls on UnionArray as cannot infer type". Any query like: ```sql SELECT CASE WHEN condition THEN returns_union(col, 'key') END FROM table ``` would panic if `condition` is false for any row. ## Root Cause The `extend_nulls` implementation for Union arrays unconditionally panicked because it claimed it "cannot infer type". However, the Union's field definitions (child types and type IDs) are available in the `MutableArrayData`'s data type — there's enough information to produce valid null entries by picking the first declared type_id. ## Test plan - [x] Added test for sparse union `extend_nulls` - [x] Added test for dense union `extend_nulls` - [x] Existing `test_union_dense` continues to pass - [x] All `array_transform` tests pass 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: Jeffrey Vo --- arrow-data/src/transform/mod.rs | 4 +- arrow-data/src/transform/union.rs | 41 ++++++++++++-- arrow/tests/array_transform.rs | 88 +++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 6 deletions(-) diff --git a/arrow-data/src/transform/mod.rs b/arrow-data/src/transform/mod.rs index c6052817bfb6..66f6603f02fc 100644 --- a/arrow-data/src/transform/mod.rs +++ b/arrow-data/src/transform/mod.rs @@ -813,8 +813,8 @@ impl<'a> MutableArrayData<'a> { }; let nulls = match data.data_type { - // RunEndEncoded and Null arrays cannot have top-level null bitmasks - DataType::RunEndEncoded(_, _) | DataType::Null => None, + // RunEndEncoded, Null, and Union arrays cannot have top-level null bitmasks + DataType::RunEndEncoded(_, _) | DataType::Null | DataType::Union(_, _) => None, _ => data .null_buffer .map(|nulls| { diff --git a/arrow-data/src/transform/union.rs b/arrow-data/src/transform/union.rs index f6f291e3f05d..d1301249d326 100644 --- a/arrow-data/src/transform/union.rs +++ b/arrow-data/src/transform/union.rs @@ -17,6 +17,7 @@ use super::{_MutableArrayData, Extend}; use crate::ArrayData; +use arrow_schema::DataType; pub(super) fn build_extend_sparse(array: &ArrayData) -> Extend<'_> { let type_ids = array.buffer::(0); @@ -68,10 +69,42 @@ pub(super) fn build_extend_dense(array: &ArrayData) -> Extend<'_> { ) } -pub(super) fn extend_nulls_dense(_mutable: &mut _MutableArrayData, _len: usize) { - panic!("cannot call extend_nulls on UnionArray as cannot infer type"); +pub(super) fn extend_nulls_dense(mutable: &mut _MutableArrayData, len: usize) { + let DataType::Union(fields, _) = &mutable.data_type else { + unreachable!() + }; + let first_type_id = fields + .iter() + .next() + .expect("union must have at least one field") + .0; + + // Extend type_ids buffer + mutable.buffer1.extend_from_slice(&vec![first_type_id; len]); + + // Dense: extend offsets pointing into the first child, then extend nulls in that child + let child_offset = mutable.child_data[0].len(); + let (start, end) = (child_offset as i32, (child_offset + len) as i32); + mutable.buffer2.extend(start..end); + mutable.child_data[0].extend_nulls(len); } -pub(super) fn extend_nulls_sparse(_mutable: &mut _MutableArrayData, _len: usize) { - panic!("cannot call extend_nulls on UnionArray as cannot infer type"); +pub(super) fn extend_nulls_sparse(mutable: &mut _MutableArrayData, len: usize) { + let DataType::Union(fields, _) = &mutable.data_type else { + unreachable!() + }; + let first_type_id = fields + .iter() + .next() + .expect("union must have at least one field") + .0; + + // Extend type_ids buffer + mutable.buffer1.extend_from_slice(&vec![first_type_id; len]); + + // Sparse: extend nulls in ALL children + mutable + .child_data + .iter_mut() + .for_each(|child| child.extend_nulls(len)); } diff --git a/arrow/tests/array_transform.rs b/arrow/tests/array_transform.rs index 511dc1e8bfcd..c24d0992a473 100644 --- a/arrow/tests/array_transform.rs +++ b/arrow/tests/array_transform.rs @@ -1151,3 +1151,91 @@ fn test_fixed_size_list_append() { .unwrap(); assert_eq!(finished, expected_fixed_size_list_data); } + +#[test] +fn test_extend_nulls_sparse_union() { + let fields = UnionFields::try_new( + vec![0, 1], + vec![ + Field::new("null", DataType::Null, true), + Field::new("str", DataType::Utf8, true), + ], + ) + .unwrap(); + + let type_ids = ScalarBuffer::from(vec![1i8]); + let child_null = Arc::new(NullArray::new(1)) as ArrayRef; + let child_str = Arc::new(StringArray::from(vec![Some("hello")])) as ArrayRef; + let union_array = UnionArray::try_new( + fields.clone(), + type_ids, + None, // sparse + vec![child_null, child_str], + ) + .unwrap(); + + let data = union_array.to_data(); + let mut mutable = MutableArrayData::new(vec![&data], true, 4); + mutable.extend(0, 0, 1); // copy the first element + mutable.extend_nulls(2); // add two nulls + let result = mutable.freeze(); + + // Union arrays must not have a null bitmap per Arrow spec + assert!(result.nulls().is_none()); + + let result_array = UnionArray::from(result); + assert_eq!(result_array.len(), 3); + // First element should be type_id 1 (str) + assert_eq!(result_array.type_id(0), 1); + // Null elements use the first type_id (0) + assert_eq!(result_array.type_id(1), 0); + assert_eq!(result_array.type_id(2), 0); + // All children should have length 3 (sparse invariant) + assert_eq!(result_array.child(0).len(), 3); + assert_eq!(result_array.child(1).len(), 3); +} + +#[test] +fn test_extend_nulls_dense_union() { + let fields = UnionFields::try_new( + vec![0, 1], + vec![ + Field::new("i", DataType::Int32, true), + Field::new("str", DataType::Utf8, true), + ], + ) + .unwrap(); + + let type_ids = ScalarBuffer::from(vec![1i8]); + let offsets = ScalarBuffer::from(vec![0i32]); + let child_int = Arc::new(Int32Array::new_null(0)) as ArrayRef; + let child_str = Arc::new(StringArray::from(vec![Some("hello")])) as ArrayRef; + let union_array = UnionArray::try_new( + fields.clone(), + type_ids, + Some(offsets), + vec![child_int, child_str], + ) + .unwrap(); + + let data = union_array.to_data(); + let mut mutable = MutableArrayData::new(vec![&data], true, 4); + mutable.extend(0, 0, 1); // copy the first element + mutable.extend_nulls(2); // add two nulls + let result = mutable.freeze(); + + // Union arrays must not have a null bitmap per Arrow spec + assert!(result.nulls().is_none()); + + let result_array = UnionArray::from(result); + assert_eq!(result_array.len(), 3); + // First element is type_id 1 (str) + assert_eq!(result_array.type_id(0), 1); + // Null elements use the first type_id (0) + assert_eq!(result_array.type_id(1), 0); + assert_eq!(result_array.type_id(2), 0); + // First child (int) should have 2 null entries from extend_nulls + assert_eq!(result_array.child(0).len(), 2); + // Second child (str) should have 1 entry from extend + assert_eq!(result_array.child(1).len(), 1); +} From 77e4d05fe0f199ccfaad578e58278329534a9c3d Mon Sep 17 00:00:00 2001 From: Liam Bao Date: Tue, 31 Mar 2026 09:59:46 -0400 Subject: [PATCH 11/17] [Json] Add json reader benchmarks for Map and REE (#9616) # Which issue does this PR close? - Relates to #9497. # Rationale for this change # What changes are included in this PR? As part of the effort to move the Json reader away from `ArrayData` toward typed `ArrayRef` APIs, it's necessary to change the `ArrayDecoder::decode` interface to return `ArrayRef` directly and updates all decoder implementations (list, struct, map, run-end encoded) to construct typed arrays without intermediate `ArrayData` round-trips. New benchmarks for map and run-end encoded decoding are added to verify there is no performance regression. # Are these changes tested? Yes # Are there any user-facing changes? No --- arrow-json/benches/json_reader.rs | 195 ++++++++++++++++++++++++++++-- 1 file changed, 184 insertions(+), 11 deletions(-) diff --git a/arrow-json/benches/json_reader.rs b/arrow-json/benches/json_reader.rs index fccac68d9bfa..9d0dd8e9a108 100644 --- a/arrow-json/benches/json_reader.rs +++ b/arrow-json/benches/json_reader.rs @@ -28,14 +28,28 @@ use std::fmt::Write; use std::hint::black_box; use std::sync::Arc; +// Shared const ROWS: usize = 1 << 17; // 128K rows const BATCH_SIZE: usize = 1 << 13; // 8K rows per batch +// Wide object / struct const WIDE_FIELDS: usize = 64; -const BINARY_BYTES: usize = 64; const WIDE_PROJECTION_TOTAL_FIELDS: usize = 100; // 100 fields total, select only 3 -const LIST_SHORT_ELEMENTS: usize = 5; -const LIST_LONG_ELEMENTS: usize = 100; + +// Binary +const BINARY_BYTES: usize = 64; + +// List +const SHORT_LIST_ELEMENTS: usize = 5; +const LONG_LIST_ELEMENTS: usize = 100; + +// Map +const SMALL_MAP_ENTRIES: usize = 5; +const LARGE_MAP_ENTRIES: usize = 50; + +// Run-end encoded +const SHORT_REE_RUN_LENGTH: usize = 2; +const LONG_REE_RUN_LENGTH: usize = 100; fn decode_and_flush(decoder: &mut Decoder, data: &[u8]) { let mut offset = 0; @@ -289,19 +303,174 @@ fn bench_decode_list(c: &mut Criterion) { let schema = build_list_schema(); // Short lists: tests list handling overhead (few elements per row) - let short_data = build_list_json(ROWS, LIST_SHORT_ELEMENTS); - bench_decode_schema(c, "decode_list_short_i64_json", &short_data, schema.clone()); + let short_data = build_list_json(ROWS, SHORT_LIST_ELEMENTS); + bench_decode_schema(c, "decode_short_list_i64_json", &short_data, schema.clone()); // Long lists: tests child element decode throughput (many elements per row) - let long_data = build_list_json(ROWS, LIST_LONG_ELEMENTS); - bench_decode_schema(c, "decode_list_long_i64_json", &long_data, schema); + let long_data = build_list_json(ROWS, LONG_LIST_ELEMENTS); + bench_decode_schema(c, "decode_long_list_i64_json", &long_data, schema); } fn bench_serialize_list(c: &mut Criterion) { let schema = build_list_schema(); - let short_values = build_list_values(ROWS, LIST_SHORT_ELEMENTS); - c.bench_function("decode_list_short_i64_serialize", |b| { + let short_values = build_list_values(ROWS, SHORT_LIST_ELEMENTS); + c.bench_function("decode_short_list_i64_serialize", |b| { + b.iter(|| { + let mut decoder = ReaderBuilder::new(schema.clone()) + .with_batch_size(BATCH_SIZE) + .build_decoder() + .unwrap(); + decoder.serialize(&short_values).unwrap(); + while let Some(_batch) = decoder.flush().unwrap() {} + }) + }); + + let long_values = build_list_values(ROWS, LONG_LIST_ELEMENTS); + c.bench_function("decode_long_list_i64_serialize", |b| { + b.iter(|| { + let mut decoder = ReaderBuilder::new(schema.clone()) + .with_batch_size(BATCH_SIZE) + .build_decoder() + .unwrap(); + decoder.serialize(&long_values).unwrap(); + while let Some(_batch) = decoder.flush().unwrap() {} + }) + }); +} + +fn build_map_json(rows: usize, entries: usize) -> Vec { + let mut out = String::with_capacity(rows * (entries * 20 + 16)); + for row in 0..rows { + out.push_str("{\"map\":{"); + for i in 0..entries { + if i > 0 { + out.push(','); + } + write!(&mut out, "\"k{}\":{}", i, (row + i) as i64).unwrap(); + } + out.push_str("}}\n"); + } + out.into_bytes() +} + +fn build_map_values(rows: usize, entries: usize) -> Vec { + let mut out = Vec::with_capacity(rows); + for row in 0..rows { + let mut inner = Map::with_capacity(entries); + for i in 0..entries { + inner.insert( + format!("k{i}"), + Value::Number(Number::from((row + i) as i64)), + ); + } + let mut map = Map::with_capacity(1); + map.insert("map".to_string(), Value::Object(inner)); + out.push(Value::Object(map)); + } + out +} + +fn build_map_schema() -> Arc { + let entries_field = Arc::new(Field::new( + "entries", + DataType::Struct( + vec![ + Field::new("keys", DataType::Utf8, false), + Field::new("values", DataType::Int64, true), + ] + .into(), + ), + false, + )); + Arc::new(Schema::new(vec![Field::new( + "map", + DataType::Map(entries_field, false), + false, + )])) +} + +fn bench_decode_map(c: &mut Criterion) { + let schema = build_map_schema(); + + let small_data = build_map_json(ROWS, SMALL_MAP_ENTRIES); + bench_decode_schema(c, "decode_small_map_json", &small_data, schema.clone()); + + let large_data = build_map_json(ROWS, LARGE_MAP_ENTRIES); + bench_decode_schema(c, "decode_large_map_json", &large_data, schema); +} + +fn bench_serialize_map(c: &mut Criterion) { + let schema = build_map_schema(); + + let small_values = build_map_values(ROWS, SMALL_MAP_ENTRIES); + c.bench_function("decode_small_map_serialize", |b| { + b.iter(|| { + let mut decoder = ReaderBuilder::new(schema.clone()) + .with_batch_size(BATCH_SIZE) + .build_decoder() + .unwrap(); + decoder.serialize(&small_values).unwrap(); + while let Some(_batch) = decoder.flush().unwrap() {} + }) + }); + + let large_values = build_map_values(ROWS, LARGE_MAP_ENTRIES); + c.bench_function("decode_large_map_serialize", |b| { + b.iter(|| { + let mut decoder = ReaderBuilder::new(schema.clone()) + .with_batch_size(BATCH_SIZE) + .build_decoder() + .unwrap(); + decoder.serialize(&large_values).unwrap(); + while let Some(_batch) = decoder.flush().unwrap() {} + }) + }); +} + +fn build_ree_json(rows: usize, run_length: usize) -> Vec { + let mut out = String::with_capacity(rows * 24); + for row in 0..rows { + let value = (row / run_length) as i64; + writeln!(&mut out, "{{\"val\":{value}}}").unwrap(); + } + out.into_bytes() +} + +fn build_ree_values(rows: usize, run_length: usize) -> Vec { + let mut out = Vec::with_capacity(rows); + for row in 0..rows { + let value = (row / run_length) as i64; + let mut map = Map::with_capacity(1); + map.insert("val".to_string(), Value::Number(Number::from(value))); + out.push(Value::Object(map)); + } + out +} + +fn build_ree_schema() -> Arc { + let ree_type = DataType::RunEndEncoded( + Arc::new(Field::new("run_ends", DataType::Int32, false)), + Arc::new(Field::new("values", DataType::Int64, true)), + ); + Arc::new(Schema::new(vec![Field::new("val", ree_type, false)])) +} + +fn bench_decode_ree(c: &mut Criterion) { + let schema = build_ree_schema(); + + let short_data = build_ree_json(ROWS, SHORT_REE_RUN_LENGTH); + bench_decode_schema(c, "decode_short_ree_runs_json", &short_data, schema.clone()); + + let long_data = build_ree_json(ROWS, LONG_REE_RUN_LENGTH); + bench_decode_schema(c, "decode_long_ree_runs_json", &long_data, schema); +} + +fn bench_serialize_ree(c: &mut Criterion) { + let schema = build_ree_schema(); + + let short_values = build_ree_values(ROWS, SHORT_REE_RUN_LENGTH); + c.bench_function("decode_short_ree_runs_serialize", |b| { b.iter(|| { let mut decoder = ReaderBuilder::new(schema.clone()) .with_batch_size(BATCH_SIZE) @@ -312,8 +481,8 @@ fn bench_serialize_list(c: &mut Criterion) { }) }); - let long_values = build_list_values(ROWS, LIST_LONG_ELEMENTS); - c.bench_function("decode_list_long_i64_serialize", |b| { + let long_values = build_ree_values(ROWS, LONG_REE_RUN_LENGTH); + c.bench_function("decode_long_ree_runs_serialize", |b| { b.iter(|| { let mut decoder = ReaderBuilder::new(schema.clone()) .with_batch_size(BATCH_SIZE) @@ -402,6 +571,10 @@ criterion_group!( bench_wide_projection, bench_decode_list, bench_serialize_list, + bench_decode_map, + bench_serialize_map, + bench_decode_ree, + bench_serialize_ree, bench_schema_inference ); criterion_main!(benches); From 1a169cd638aa4b72ccb4961e37e5014a66308718 Mon Sep 17 00:00:00 2001 From: Alexander Rafferty Date: Wed, 1 Apr 2026 06:27:16 +1100 Subject: [PATCH 12/17] Fix `MutableBuffer::clear` (#9622) # Which issue does this PR close? - closes https://github.com/apache/arrow-rs/pull/9593 # Rationale for this change In a previous PR (#9593), I change instances of `truncate(0)` to `clear()`. However, this breaks the test `test_truncate_with_pool` at `arrow-buffer/src/buffer/mutable.rs:1357`, due to an inconsistency between the implementation of `truncate` and `clear`. This PR fixes that test. # What changes are included in this PR? This PR copies a section of code related to the `pool` feature present in `truncate` but absent in `clear`, fixing the failing unit test. # Are these changes tested? Yes. # Are there any user-facing changes? No. --- arrow-buffer/src/buffer/mutable.rs | 10 ++++++++-- arrow-json/src/reader/value_iter.rs | 2 +- parquet/tests/geospatial.rs | 4 ++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/arrow-buffer/src/buffer/mutable.rs b/arrow-buffer/src/buffer/mutable.rs index 9fc860506194..07ef965cb082 100644 --- a/arrow-buffer/src/buffer/mutable.rs +++ b/arrow-buffer/src/buffer/mutable.rs @@ -450,7 +450,13 @@ impl MutableBuffer { /// Clear all existing data from this buffer. pub fn clear(&mut self) { - self.len = 0 + self.len = 0; + #[cfg(feature = "pool")] + { + if let Some(reservation) = self.reservation.lock().unwrap().as_mut() { + reservation.resize(self.len); + } + } } /// Returns the data stored in this buffer as a slice. @@ -1371,7 +1377,7 @@ mod tests { assert_eq!(pool.used(), 40); // Truncate to zero - buffer.truncate(0); + buffer.clear(); assert_eq!(buffer.len(), 0); assert_eq!(pool.used(), 0); } diff --git a/arrow-json/src/reader/value_iter.rs b/arrow-json/src/reader/value_iter.rs index f70b893f52a0..ebaba695adf3 100644 --- a/arrow-json/src/reader/value_iter.rs +++ b/arrow-json/src/reader/value_iter.rs @@ -73,7 +73,7 @@ impl Iterator for ValueIter { } loop { - self.line_buf.truncate(0); + self.line_buf.clear(); match self.reader.read_line(&mut self.line_buf) { Ok(0) => { // read_line returns 0 when stream reached EOF diff --git a/parquet/tests/geospatial.rs b/parquet/tests/geospatial.rs index 4f449df920e8..fcc93661ed97 100644 --- a/parquet/tests/geospatial.rs +++ b/parquet/tests/geospatial.rs @@ -380,8 +380,8 @@ mod test { for i in 0..reader.num_row_groups() { let row_group = reader.get_row_group(i).unwrap(); - values.truncate(0); - def_levels.truncate(0); + values.clear(); + def_levels.clear(); let mut row_group_out = writer.next_row_group().unwrap(); From f91231160716d2b726a6bd01ef1b596c9ff69e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kriszti=C3=A1n=20Sz=C5=B1cs?= Date: Tue, 31 Mar 2026 21:31:35 +0200 Subject: [PATCH 13/17] feat(parquet): derive `PartialEq` and `Eq` for `CdcOptions` (#9602) # Rationale for this change CdcOptions only contains primitive fields (usize, usize, i32) so deriving PartialEq and Eq is straightforward. This is needed by downstream crates such as DataFusion that embed CdcOptions in their own configuration structs and need to compare them. # What changes are included in this PR? Implemented PartialEq and Eq for CdcOptions. # Are these changes tested? Added an equality test. # Are there any user-facing changes? No. --- parquet/src/file/properties.rs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/parquet/src/file/properties.rs b/parquet/src/file/properties.rs index ae15cc6b8263..640a7a075d2f 100644 --- a/parquet/src/file/properties.rs +++ b/parquet/src/file/properties.rs @@ -80,7 +80,7 @@ pub const DEFAULT_CDC_NORM_LEVEL: i32 = 0; /// following options control the chunks' size and the chunking process. Note /// that the chunk size is calculated based on the logical value of the data, /// before any encoding or compression is applied. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct CdcOptions { /// Minimum chunk size in bytes, default is 256 KiB. /// The rolling hash will not be updated until this size is reached for each chunk. @@ -1864,4 +1864,18 @@ mod tests { } } } + + #[test] + fn test_cdc_options_equality() { + let opts = CdcOptions::default(); + assert_eq!(opts, CdcOptions::default()); + + let custom = CdcOptions { + min_chunk_size: 1024, + max_chunk_size: 8192, + norm_level: 1, + }; + assert_eq!(custom, custom); + assert_ne!(opts, custom); + } } From 51bf8a40f72e37528cf36419f8f453ccd0e45868 Mon Sep 17 00:00:00 2001 From: Konstantin Tarasov <33369833+sdf-jkl@users.noreply.github.com> Date: Tue, 31 Mar 2026 15:44:32 -0400 Subject: [PATCH 14/17] [Variant] extend shredded null handling for arrays (#9599) # Which issue does this PR close? - Closes #8400. # Rationale for this change Check issue # What changes are included in this PR? - Added `AppendNullMode` enum supporting all semantics. - Replaced the bool logic to the new enum - Fix test outputs for List Array cases # Are these changes tested? - Added unit tests # Are there any user-facing changes? --- parquet-variant-compute/src/shred_variant.rs | 242 ++++++++++++++---- .../src/type_conversion.rs | 24 +- parquet-variant-compute/src/variant_get.rs | 53 ++++ .../src/variant_to_arrow.rs | 238 ++++++++++++----- 4 files changed, 436 insertions(+), 121 deletions(-) diff --git a/parquet-variant-compute/src/shred_variant.rs b/parquet-variant-compute/src/shred_variant.rs index 6520ea700b0c..d80d2f9863f6 100644 --- a/parquet-variant-compute/src/shred_variant.rs +++ b/parquet-variant-compute/src/shred_variant.rs @@ -84,7 +84,7 @@ pub fn shred_variant(array: &VariantArray, as_type: &DataType) -> Result Result nulls.append_null(), + Self::ObjectField | Self::ArrayElement => nulls.append_non_null(), + } + match self { + Self::TopLevelVariant | Self::ObjectField => value_builder.append_null(), + Self::ArrayElement => value_builder.append_value(Variant::Null), + } + } +} + pub(crate) fn make_variant_to_shredded_variant_arrow_row_builder<'a>( data_type: &'a DataType, cast_options: &'a CastOptions, capacity: usize, - top_level: bool, + null_value: NullValue, ) -> Result> { let builder = match data_type { DataType::Struct(fields) => { @@ -114,7 +145,7 @@ pub(crate) fn make_variant_to_shredded_variant_arrow_row_builder<'a>( fields, cast_options, capacity, - top_level, + null_value, )?; VariantToShreddedVariantRowBuilder::Object(typed_value_builder) } @@ -127,6 +158,7 @@ pub(crate) fn make_variant_to_shredded_variant_arrow_row_builder<'a>( data_type, cast_options, capacity, + null_value, )?; VariantToShreddedVariantRowBuilder::Array(typed_value_builder) } @@ -156,7 +188,7 @@ pub(crate) fn make_variant_to_shredded_variant_arrow_row_builder<'a>( let builder = make_primitive_variant_to_arrow_row_builder(data_type, cast_options, capacity)?; let typed_value_builder = - VariantToShreddedPrimitiveVariantRowBuilder::new(builder, capacity, top_level); + VariantToShreddedPrimitiveVariantRowBuilder::new(builder, capacity, null_value); VariantToShreddedVariantRowBuilder::Primitive(typed_value_builder) } DataType::FixedSizeBinary(_) => { @@ -204,33 +236,31 @@ impl<'a> VariantToShreddedVariantRowBuilder<'a> { } } -/// A top-level variant shredder -- appending NULL produces typed_value=NULL and value=Variant::Null +/// A shredded primitive field builder. pub(crate) struct VariantToShreddedPrimitiveVariantRowBuilder<'a> { value_builder: VariantValueArrayBuilder, typed_value_builder: PrimitiveVariantToArrowRowBuilder<'a>, nulls: NullBufferBuilder, - top_level: bool, + null_value: NullValue, } impl<'a> VariantToShreddedPrimitiveVariantRowBuilder<'a> { pub(crate) fn new( typed_value_builder: PrimitiveVariantToArrowRowBuilder<'a>, capacity: usize, - top_level: bool, + null_value: NullValue, ) -> Self { Self { value_builder: VariantValueArrayBuilder::new(capacity), typed_value_builder, nulls: NullBufferBuilder::new(capacity), - top_level, + null_value, } } fn append_null(&mut self) -> Result<()> { - // Only the top-level struct that represents the variant can be nullable; object fields and - // array elements are non-nullable. - self.nulls.append(!self.top_level); - self.value_builder.append_null(); + self.null_value + .append_to(&mut self.nulls, &mut self.value_builder); self.typed_value_builder.append_null() } @@ -256,6 +286,8 @@ impl<'a> VariantToShreddedPrimitiveVariantRowBuilder<'a> { pub(crate) struct VariantToShreddedArrayVariantRowBuilder<'a> { value_builder: VariantValueArrayBuilder, typed_value_builder: ArrayVariantToArrowRowBuilder<'a>, + nulls: NullBufferBuilder, + null_value: NullValue, } impl<'a> VariantToShreddedArrayVariantRowBuilder<'a> { @@ -263,6 +295,7 @@ impl<'a> VariantToShreddedArrayVariantRowBuilder<'a> { data_type: &'a DataType, cast_options: &'a CastOptions, capacity: usize, + null_value: NullValue, ) -> Result { Ok(Self { value_builder: VariantValueArrayBuilder::new(capacity), @@ -271,11 +304,14 @@ impl<'a> VariantToShreddedArrayVariantRowBuilder<'a> { cast_options, capacity, )?, + nulls: NullBufferBuilder::new(capacity), + null_value, }) } fn append_null(&mut self) -> Result<()> { - self.value_builder.append_value(Variant::Null); + self.null_value + .append_to(&mut self.nulls, &mut self.value_builder); self.typed_value_builder.append_null()?; Ok(()) } @@ -285,12 +321,14 @@ impl<'a> VariantToShreddedArrayVariantRowBuilder<'a> { // If the variant is an array, value must be null. match variant { Variant::List(list) => { + self.nulls.append_non_null(); self.value_builder.append_null(); self.typed_value_builder .append_value(&Variant::List(list))?; Ok(true) } other => { + self.nulls.append_non_null(); self.value_builder.append_value(other); self.typed_value_builder.append_null()?; Ok(false) @@ -298,13 +336,11 @@ impl<'a> VariantToShreddedArrayVariantRowBuilder<'a> { } } - fn finish(self) -> Result<(BinaryViewArray, ArrayRef, Option)> { + fn finish(mut self) -> Result<(BinaryViewArray, ArrayRef, Option)> { Ok(( self.value_builder.build()?, self.typed_value_builder.finish()?, - // All elements of an array must be present (not missing) because - // the array Variant encoding does not allow missing elements - None, + self.nulls.finish(), )) } } @@ -314,7 +350,7 @@ pub(crate) struct VariantToShreddedObjectVariantRowBuilder<'a> { typed_value_builders: IndexMap<&'a str, VariantToShreddedVariantRowBuilder<'a>>, typed_value_nulls: NullBufferBuilder, nulls: NullBufferBuilder, - top_level: bool, + null_value: NullValue, } impl<'a> VariantToShreddedObjectVariantRowBuilder<'a> { @@ -322,14 +358,14 @@ impl<'a> VariantToShreddedObjectVariantRowBuilder<'a> { fields: &'a Fields, cast_options: &'a CastOptions, capacity: usize, - top_level: bool, + null_value: NullValue, ) -> Result { let typed_value_builders = fields.iter().map(|field| { let builder = make_variant_to_shredded_variant_arrow_row_builder( field.data_type(), cast_options, capacity, - false, + NullValue::ObjectField, )?; Ok((field.name().as_str(), builder)) }); @@ -338,15 +374,13 @@ impl<'a> VariantToShreddedObjectVariantRowBuilder<'a> { typed_value_builders: typed_value_builders.collect::>()?, typed_value_nulls: NullBufferBuilder::new(capacity), nulls: NullBufferBuilder::new(capacity), - top_level, + null_value, }) } fn append_null(&mut self) -> Result<()> { - // Only the top-level struct that represents the variant can be nullable; object fields and - // array elements are non-nullable. - self.nulls.append(!self.top_level); - self.value_builder.append_null(); + self.null_value + .append_to(&mut self.nulls, &mut self.value_builder); self.typed_value_nulls.append_null(); for (_, typed_value_builder) in &mut self.typed_value_builders { typed_value_builder.append_null()?; @@ -669,6 +703,12 @@ mod tests { use std::sync::Arc; use uuid::Uuid; + const NULL_VALUES: [NullValue; 3] = [ + NullValue::TopLevelVariant, + NullValue::ObjectField, + NullValue::ArrayElement, + ]; + #[derive(Clone)] enum VariantValue<'a> { Value(Variant<'a, 'a>), @@ -881,7 +921,9 @@ mod tests { expected_variant.clone() ); } - None => unreachable!(), + None => { + assert!(fallbacks.0.is_null(idx)); + } } } } @@ -949,6 +991,121 @@ mod tests { } } + fn assert_append_null_mode_value_and_struct_nulls( + mode: NullValue, + value: &BinaryViewArray, + nulls: Option<&arrow::buffer::NullBuffer>, + ) { + if mode == NullValue::TopLevelVariant { + assert!(nulls.is_some_and(|n| n.is_null(0))); + } else { + assert!(nulls.is_none()); + } + + if mode == NullValue::ArrayElement { + assert!(value.is_valid(0)); + assert_eq!( + Variant::new(EMPTY_VARIANT_METADATA_BYTES, value.value(0)), + Variant::Null + ); + } else { + assert!(value.is_null(0)); + } + } + + #[test] + fn test_append_null_mode_semantics_primitive_builder() { + let cast_options = arrow::compute::CastOptions::default(); + + for mode in NULL_VALUES { + let mut primitive_builder = make_variant_to_shredded_variant_arrow_row_builder( + &DataType::Int64, + &cast_options, + 1, + mode, + ) + .unwrap(); + primitive_builder.append_null().unwrap(); + let (primitive_value, primitive_typed_value, primitive_nulls) = + primitive_builder.finish().unwrap(); + let primitive_typed_value = primitive_typed_value + .as_any() + .downcast_ref::() + .unwrap(); + + assert!(primitive_typed_value.is_null(0)); + assert_append_null_mode_value_and_struct_nulls( + mode, + &primitive_value, + primitive_nulls.as_ref(), + ); + } + } + + #[test] + fn test_append_null_mode_semantics_array_builder() { + let cast_options = arrow::compute::CastOptions::default(); + let list_type = DataType::List(Arc::new(Field::new("item", DataType::Int64, true))); + + for mode in NULL_VALUES { + let mut array_builder = make_variant_to_shredded_variant_arrow_row_builder( + &list_type, + &cast_options, + 1, + mode, + ) + .unwrap(); + array_builder.append_null().unwrap(); + let (value, typed_value, nulls) = array_builder.finish().unwrap(); + + assert_append_null_mode_value_and_struct_nulls(mode, &value, nulls.as_ref()); + + let typed_value = typed_value.as_any().downcast_ref::().unwrap(); + assert_eq!(typed_value.len(), 1); + assert!(typed_value.is_null(0)); + assert_eq!(typed_value.values().len(), 0); + } + } + + #[test] + fn test_append_null_mode_semantics_object_builder() { + let cast_options = arrow::compute::CastOptions::default(); + let object_type = DataType::Struct(Fields::from(vec![ + Field::new("id", DataType::Int64, true), + Field::new("name", DataType::Utf8, true), + ])); + + for mode in NULL_VALUES { + let mut object_builder = make_variant_to_shredded_variant_arrow_row_builder( + &object_type, + &cast_options, + 1, + mode, + ) + .unwrap(); + object_builder.append_null().unwrap(); + let (value, typed_value, nulls) = object_builder.finish().unwrap(); + + assert_append_null_mode_value_and_struct_nulls(mode, &value, nulls.as_ref()); + + let typed_struct = typed_value + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(typed_struct.len(), 1); + assert!(typed_struct.is_null(0)); + + for field_name in ["id", "name"] { + let field = ShreddedVariantFieldArray::try_new( + typed_struct.column_by_name(field_name).unwrap(), + ) + .unwrap(); + assert!(field.value_field().unwrap().is_null(0)); + assert!(field.typed_value_field().unwrap().is_null(0)); + } + } + } + #[test] fn test_already_shredded_input_error() { // Create a VariantArray that already has typed_value_field @@ -1338,13 +1495,7 @@ mod tests { 5, &[0, 3, 6, 6, 6, 6], &[Some(3), Some(3), None, None, Some(0)], - &[ - None, - None, - Some(Variant::from("not a list")), - Some(Variant::Null), - None, - ], + &[None, None, Some(Variant::from("not a list")), None, None], ( &[Some(1), Some(2), Some(3), Some(1), None, None], &[ @@ -1414,13 +1565,7 @@ mod tests { 5, &[0, 3, 6, 6, 6], &[Some(3), Some(3), None, None, Some(0)], - &[ - None, - None, - Some(Variant::from("not a list")), - Some(Variant::Null), - None, - ], + &[None, None, Some(Variant::from("not a list")), None, None], ( &[Some(1), Some(2), Some(3), Some(1), None, None], &[ @@ -1522,12 +1667,7 @@ mod tests { 4, &[0, 3, 6, 6, 6], &[Some(3), Some(3), None, None], - &[ - None, - None, - Some(Variant::from("not a list")), - Some(Variant::Null), - ], + &[None, None, Some(Variant::from("not a list")), None], ); let outer_elements = @@ -1615,7 +1755,7 @@ mod tests { 3, &[0, 2, 2, 2], &[Some(2), None, None], - &[None, Some(Variant::from("not a list")), Some(Variant::Null)], + &[None, Some(Variant::from("not a list")), None], ); // Validate nested struct fields for each element @@ -2101,13 +2241,7 @@ mod tests { scores_field.len(), &[0i32, 2, 4, 4, 4, 4], &[Some(2), Some(2), None, None, None], - &[ - None, - None, - Some(Variant::Null), - Some(Variant::Null), - Some(Variant::Null), - ], + &[None, None, None, None, None], ( &[Some(10), Some(20), None, None], &[None, None, Some(Variant::from("oops")), Some(Variant::Null)], diff --git a/parquet-variant-compute/src/type_conversion.rs b/parquet-variant-compute/src/type_conversion.rs index 4086a2410792..7b9eb67d1a95 100644 --- a/parquet-variant-compute/src/type_conversion.rs +++ b/parquet-variant-compute/src/type_conversion.rs @@ -17,11 +17,12 @@ //! Module for transforming a typed arrow `Array` to `VariantArray`. -use arrow::compute::{DecimalCast, rescale_decimal}; +use arrow::compute::{CastOptions, DecimalCast, rescale_decimal}; use arrow::datatypes::{ self, ArrowPrimitiveType, ArrowTimestampType, Decimal32Type, Decimal64Type, Decimal128Type, DecimalType, }; +use arrow::error::{ArrowError, Result}; use chrono::Timelike; use parquet_variant::{Variant, VariantDecimal4, VariantDecimal8, VariantDecimal16}; @@ -37,6 +38,27 @@ pub(crate) trait TimestampFromVariant: ArrowTimestampType { fn from_variant(variant: &Variant<'_, '_>) -> Option; } +/// Cast a single `Variant` value with safe/strict semantics. +/// +/// Returns `Ok(Some(_))` on successful conversion. +/// Returns `Ok(None)` when conversion fails in safe mode or the source value is `Variant::Null`. +/// Returns `Err(_)` when conversion fails in strict mode. +pub(crate) fn variant_cast_with_options<'a, 'm, 'v, T>( + variant: &'a Variant<'m, 'v>, + cast_options: &CastOptions<'_>, + cast: impl FnOnce(&'a Variant<'m, 'v>) -> Option, +) -> Result> { + if let Some(value) = cast(variant) { + Ok(Some(value)) + } else if matches!(variant, Variant::Null) || cast_options.safe { + Ok(None) + } else { + Err(ArrowError::CastError(format!( + "Failed to cast variant value {variant:?}" + ))) + } +} + /// Macro to generate PrimitiveFromVariant implementations for Arrow primitive types macro_rules! impl_primitive_from_variant { ($arrow_type:ty, $variant_method:ident $(, $cast_fn:expr)?) => { diff --git a/parquet-variant-compute/src/variant_get.rs b/parquet-variant-compute/src/variant_get.rs index 3e9892cacf70..73906f70eb77 100644 --- a/parquet-variant-compute/src/variant_get.rs +++ b/parquet-variant-compute/src/variant_get.rs @@ -4270,6 +4270,59 @@ mod test { } } + #[test] + fn test_variant_get_list_like_unsafe_cast_preserves_null_elements() { + let string_array: ArrayRef = Arc::new(StringArray::from(vec![r#"[1, null, 3]"#])); + let variant_array = ArrayRef::from(json_to_variant(&string_array).unwrap()); + let cast_options = CastOptions { + safe: false, + ..Default::default() + }; + let options = GetOptions::new() + .with_as_type(Some(FieldRef::from(Field::new( + "result", + DataType::List(Arc::new(Field::new("item", DataType::Int64, true))), + true, + )))) + .with_cast_options(cast_options); + + let result = variant_get(&variant_array, options).unwrap(); + let element_struct = result + .as_any() + .downcast_ref::() + .unwrap() + .values() + .as_any() + .downcast_ref::() + .unwrap(); + + let value = element_struct + .column_by_name("value") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let typed_value = element_struct + .column_by_name("typed_value") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + + assert_eq!(typed_value.len(), 3); + assert_eq!(typed_value.value(0), 1); + assert!(typed_value.is_null(1)); + assert_eq!(typed_value.value(2), 3); + + assert!(value.is_null(0)); + assert!(value.is_valid(1)); + assert_eq!( + Variant::new(EMPTY_VARIANT_METADATA_BYTES, value.value(1)), + Variant::Null + ); + assert!(value.is_null(2)); + } + #[test] fn test_variant_get_list_like_unsafe_cast_errors_on_non_list() { let string_array: ArrayRef = Arc::new(StringArray::from(vec!["[1, 2]", "\"not a list\""])); diff --git a/parquet-variant-compute/src/variant_to_arrow.rs b/parquet-variant-compute/src/variant_to_arrow.rs index dc8fbcd223d2..dd396117d22d 100644 --- a/parquet-variant-compute/src/variant_to_arrow.rs +++ b/parquet-variant-compute/src/variant_to_arrow.rs @@ -16,10 +16,12 @@ // under the License. use crate::shred_variant::{ - VariantToShreddedVariantRowBuilder, make_variant_to_shredded_variant_arrow_row_builder, + NullValue, VariantToShreddedVariantRowBuilder, + make_variant_to_shredded_variant_arrow_row_builder, }; use crate::type_conversion::{ - PrimitiveFromVariant, TimestampFromVariant, variant_to_unscaled_decimal, + PrimitiveFromVariant, TimestampFromVariant, variant_cast_with_options, + variant_to_unscaled_decimal, }; use crate::variant_array::ShreddedVariantFieldArray; use crate::{VariantArray, VariantValueArrayBuilder}; @@ -545,30 +547,30 @@ impl<'a> StructVariantToArrowRowBuilder<'a> { } fn append_value(&mut self, value: &Variant<'_, '_>) -> Result { - let Variant::Object(obj) = value else { - if self.cast_options.safe { - self.append_null()?; - return Ok(false); - } - return Err(ArrowError::CastError(format!( - "Failed to extract struct from variant {:?}", - value - ))); - }; - - for (index, field) in self.fields.iter().enumerate() { - match obj.get(field.name()) { - Some(field_value) => { - self.field_builders[index].append_value(field_value)?; - } - None => { - self.field_builders[index].append_null()?; + match variant_cast_with_options(value, self.cast_options, Variant::as_object) { + Ok(Some(obj)) => { + for (index, field) in self.fields.iter().enumerate() { + match obj.get(field.name()) { + Some(field_value) => { + self.field_builders[index].append_value(field_value)?; + } + None => { + self.field_builders[index].append_null()?; + } + } } + + self.nulls.append_non_null(); + Ok(true) + } + Ok(None) => { + self.append_null()?; + Ok(false) } + Err(_) => Err(ArrowError::CastError(format!( + "Failed to extract struct from variant {value:?}" + ))), } - - self.nulls.append_non_null(); - Ok(true) } fn finish(mut self) -> Result { @@ -707,21 +709,24 @@ macro_rules! define_variant_to_primitive_builder { } fn append_value(&mut self, $value: &Variant<'_, '_>) -> Result { - if let Some(v) = $value_transform { - self.builder.append_value(v); - Ok(true) - } else { - if !self.cast_options.safe { - // Unsafe casting: return error on conversion failure - return Err(ArrowError::CastError(format!( - "Failed to extract primitive of type {} from variant {:?} at path VariantPath([])", - $type_name, - $value - ))); + match variant_cast_with_options( + $value, + self.cast_options, + |$value| $value_transform, + ) { + Ok(Some(v)) => { + self.builder.append_value(v); + Ok(true) + } + Ok(None) => { + self.builder.append_null(); + Ok(false) } - // Safe casting: append null on conversion failure - self.builder.append_null(); - Ok(false) + Err(_) => Err(ArrowError::CastError(format!( + "Failed to extract primitive of type {type_name} from variant {value:?} at path VariantPath([])", + type_name = $type_name, + value = $value + ))), } } @@ -748,7 +753,7 @@ define_variant_to_primitive_builder!( define_variant_to_primitive_builder!( struct VariantToBooleanArrowRowBuilder<'a> |capacity| -> BooleanBuilder { BooleanBuilder::with_capacity(capacity) }, - |value| value.as_boolean(), + |value| value.as_boolean(), type_name: datatypes::BooleanType::DATA_TYPE ); @@ -821,20 +826,23 @@ where } fn append_value(&mut self, value: &Variant<'_, '_>) -> Result { - if let Some(scaled) = variant_to_unscaled_decimal::(value, self.precision, self.scale) { - self.builder.append_value(scaled); - Ok(true) - } else if self.cast_options.safe { - self.builder.append_null(); - Ok(false) - } else { - Err(ArrowError::CastError(format!( - "Failed to cast to {}(precision={}, scale={}) from variant {:?}", - T::PREFIX, - self.precision, - self.scale, - value - ))) + match variant_cast_with_options(value, self.cast_options, |value| { + variant_to_unscaled_decimal::(value, self.precision, self.scale) + }) { + Ok(Some(scaled)) => { + self.builder.append_value(scaled); + Ok(true) + } + Ok(None) => { + self.builder.append_null(); + Ok(false) + } + Err(_) => Err(ArrowError::CastError(format!( + "Failed to cast to {prefix}(precision={precision}, scale={scale}) from variant {value:?}", + prefix = T::PREFIX, + precision = self.precision, + scale = self.scale + ))), } } @@ -863,20 +871,19 @@ impl<'a> VariantToUuidArrowRowBuilder<'a> { } fn append_value(&mut self, value: &Variant<'_, '_>) -> Result { - match value.as_uuid() { - Some(uuid) => { + match variant_cast_with_options(value, self.cast_options, Variant::as_uuid) { + Ok(Some(uuid)) => { self.builder .append_value(uuid.as_bytes()) .map_err(|e| ArrowError::ExternalError(Box::new(e)))?; - Ok(true) } - None if self.cast_options.safe => { + Ok(None) => { self.builder.append_null(); Ok(false) } - None => Err(ArrowError::CastError(format!( - "Failed to extract UUID from variant {value:?}", + Err(_) => Err(ArrowError::CastError(format!( + "Failed to extract UUID from variant {value:?}" ))), } } @@ -919,7 +926,7 @@ where element_data_type, cast_options, capacity, - false, + NullValue::ArrayElement, )?; Ok(Self { field, @@ -938,8 +945,8 @@ where } fn append_value(&mut self, value: &Variant<'_, '_>) -> Result { - match value { - Variant::List(list) => { + match variant_cast_with_options(value, self.cast_options, Variant::as_list) { + Ok(Some(list)) => { for element in list.iter() { self.element_builder.append_value(element)?; self.current_offset = self.current_offset.add_checked(O::ONE)?; @@ -948,13 +955,12 @@ where self.nulls.append_non_null(); Ok(true) } - _ if self.cast_options.safe => { + Ok(None) => { self.append_null()?; Ok(false) } - _ => Err(ArrowError::CastError(format!( - "Failed to extract list from variant {:?}", - value + Err(_) => Err(ArrowError::CastError(format!( + "Failed to extract list from variant {value:?}" ))), } } @@ -1067,11 +1073,18 @@ define_variant_to_primitive_builder!( #[cfg(test)] mod tests { - use super::make_primitive_variant_to_arrow_row_builder; + use super::{ + make_primitive_variant_to_arrow_row_builder, make_typed_variant_to_arrow_row_builder, + }; + use arrow::array::{ + Array, Decimal32Array, FixedSizeBinaryArray, Int32Array, ListArray, StructArray, + }; use arrow::compute::CastOptions; use arrow::datatypes::{DataType, Field, Fields, UnionFields, UnionMode}; use arrow::error::ArrowError; + use parquet_variant::{Variant, VariantDecimal4}; use std::sync::Arc; + use uuid::Uuid; #[test] fn make_primitive_builder_rejects_non_primitive_types() { @@ -1120,4 +1133,97 @@ mod tests { } } } + + #[test] + fn strict_cast_allows_variant_null_for_primitive_builder() { + let cast_options = CastOptions { + safe: false, + ..Default::default() + }; + let mut builder = + make_primitive_variant_to_arrow_row_builder(&DataType::Int32, &cast_options, 2) + .unwrap(); + + assert!(!builder.append_value(&Variant::Null).unwrap()); + assert!(builder.append_value(&Variant::Int32(42)).unwrap()); + + let array = builder.finish().unwrap(); + let int_array = array.as_any().downcast_ref::().unwrap(); + assert!(int_array.is_null(0)); + assert_eq!(int_array.value(1), 42); + } + + #[test] + fn strict_cast_allows_variant_null_for_decimal_builder() { + let cast_options = CastOptions { + safe: false, + ..Default::default() + }; + let mut builder = make_primitive_variant_to_arrow_row_builder( + &DataType::Decimal32(9, 2), + &cast_options, + 2, + ) + .unwrap(); + let decimal_variant: Variant<'_, '_> = VariantDecimal4::try_new(1234, 2).unwrap().into(); + + assert!(!builder.append_value(&Variant::Null).unwrap()); + assert!(builder.append_value(&decimal_variant).unwrap()); + + let array = builder.finish().unwrap(); + let decimal_array = array.as_any().downcast_ref::().unwrap(); + assert!(decimal_array.is_null(0)); + assert_eq!(decimal_array.value(1), 1234); + } + + #[test] + fn strict_cast_allows_variant_null_for_uuid_builder() { + let cast_options = CastOptions { + safe: false, + ..Default::default() + }; + let mut builder = make_primitive_variant_to_arrow_row_builder( + &DataType::FixedSizeBinary(16), + &cast_options, + 2, + ) + .unwrap(); + let uuid = Uuid::nil(); + + assert!(!builder.append_value(&Variant::Null).unwrap()); + assert!(builder.append_value(&Variant::Uuid(uuid)).unwrap()); + + let array = builder.finish().unwrap(); + let uuid_array = array + .as_any() + .downcast_ref::() + .unwrap(); + assert!(uuid_array.is_null(0)); + assert_eq!(uuid_array.value(1), uuid.as_bytes()); + } + + #[test] + fn strict_cast_allows_variant_null_for_list_and_struct_builders() { + let cast_options = CastOptions { + safe: false, + ..Default::default() + }; + + let list_type = DataType::List(Arc::new(Field::new("item", DataType::Int64, true))); + let mut list_builder = + make_typed_variant_to_arrow_row_builder(&list_type, &cast_options, 1).unwrap(); + assert!(!list_builder.append_value(Variant::Null).unwrap()); + let list_array = list_builder.finish().unwrap(); + let list_array = list_array.as_any().downcast_ref::().unwrap(); + assert!(list_array.is_null(0)); + + let struct_type = + DataType::Struct(Fields::from(vec![Field::new("a", DataType::Int32, true)])); + let mut struct_builder = + make_typed_variant_to_arrow_row_builder(&struct_type, &cast_options, 1).unwrap(); + assert!(!struct_builder.append_value(Variant::Null).unwrap()); + let struct_array = struct_builder.finish().unwrap(); + let struct_array = struct_array.as_any().downcast_ref::().unwrap(); + assert!(struct_array.is_null(0)); + } } From 61b5763a368db4bfa76e8fbafdbf26718f39a031 Mon Sep 17 00:00:00 2001 From: Thomas Tanon Date: Tue, 31 Mar 2026 22:40:44 +0200 Subject: [PATCH 15/17] pyarrow: Small code simplifications (#9594) # Rationale for this change Makes the code simpler and more readable by relying on new PyO3 and Rust features. No behavior should have changed outside of an error message if `__arrow_c_array__` does not return a tuple # What changes are included in this PR? - use `.call_method0(M)?` instead of `.getattr(M)?.call0()` - Use `.extract()` that allows more advanced features like directly extracting tuple elements - remove temporary variables just before returning - use &raw const and &raw mut pointers instead of casting and addr_of! --- arrow-pyarrow/src/lib.rs | 155 ++++++++++++--------------------------- 1 file changed, 48 insertions(+), 107 deletions(-) diff --git a/arrow-pyarrow/src/lib.rs b/arrow-pyarrow/src/lib.rs index e396711f873d..95f1d38fddf3 100644 --- a/arrow-pyarrow/src/lib.rs +++ b/arrow-pyarrow/src/lib.rs @@ -61,7 +61,6 @@ use std::convert::{From, TryFrom}; use std::ffi::CStr; -use std::ptr::{addr_of, addr_of_mut}; use std::sync::Arc; use arrow_array::ffi; @@ -156,36 +155,27 @@ impl FromPyArrow for DataType { // method, so prefer it over _export_to_c. // See https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html if value.hasattr("__arrow_c_schema__")? { - let capsule = value.getattr("__arrow_c_schema__")?.call0()?; - let capsule = capsule.cast::()?; - validate_pycapsule(capsule, "arrow_schema")?; + let capsule = value.call_method0("__arrow_c_schema__")?.extract()?; + validate_pycapsule(&capsule, "arrow_schema")?; let schema_ptr = capsule .pointer_checked(Some(ARROW_SCHEMA_CAPSULE_NAME))? .cast::(); - unsafe { - let dtype = DataType::try_from(schema_ptr.as_ref()).map_err(to_py_err)?; - return Ok(dtype); - } + return unsafe { DataType::try_from(schema_ptr.as_ref()) }.map_err(to_py_err); } validate_class(data_type_class(value.py())?, value)?; - let c_schema = FFI_ArrowSchema::empty(); - let c_schema_ptr = &c_schema as *const FFI_ArrowSchema; - value.call_method1("_export_to_c", (c_schema_ptr as Py_uintptr_t,))?; - let dtype = DataType::try_from(&c_schema).map_err(to_py_err)?; - Ok(dtype) + let mut c_schema = FFI_ArrowSchema::empty(); + value.call_method1("_export_to_c", (&raw mut c_schema as Py_uintptr_t,))?; + DataType::try_from(&c_schema).map_err(to_py_err) } } impl ToPyArrow for DataType { fn to_pyarrow<'py>(&self, py: Python<'py>) -> PyResult> { let c_schema = FFI_ArrowSchema::try_from(self).map_err(to_py_err)?; - let c_schema_ptr = &c_schema as *const FFI_ArrowSchema; - let dtype = - data_type_class(py)?.call_method1("_import_from_c", (c_schema_ptr as Py_uintptr_t,))?; - Ok(dtype) + data_type_class(py)?.call_method1("_import_from_c", (&raw const c_schema as Py_uintptr_t,)) } } @@ -195,36 +185,27 @@ impl FromPyArrow for Field { // method, so prefer it over _export_to_c. // See https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html if value.hasattr("__arrow_c_schema__")? { - let capsule = value.getattr("__arrow_c_schema__")?.call0()?; - let capsule = capsule.cast::()?; - validate_pycapsule(capsule, "arrow_schema")?; + let capsule = value.call_method0("__arrow_c_schema__")?.extract()?; + validate_pycapsule(&capsule, "arrow_schema")?; let schema_ptr = capsule .pointer_checked(Some(ARROW_SCHEMA_CAPSULE_NAME))? .cast::(); - unsafe { - let field = Field::try_from(schema_ptr.as_ref()).map_err(to_py_err)?; - return Ok(field); - } + return unsafe { Field::try_from(schema_ptr.as_ref()) }.map_err(to_py_err); } validate_class(field_class(value.py())?, value)?; - let c_schema = FFI_ArrowSchema::empty(); - let c_schema_ptr = &c_schema as *const FFI_ArrowSchema; - value.call_method1("_export_to_c", (c_schema_ptr as Py_uintptr_t,))?; - let field = Field::try_from(&c_schema).map_err(to_py_err)?; - Ok(field) + let mut c_schema = FFI_ArrowSchema::empty(); + value.call_method1("_export_to_c", (&raw mut c_schema as Py_uintptr_t,))?; + Field::try_from(&c_schema).map_err(to_py_err) } } impl ToPyArrow for Field { fn to_pyarrow<'py>(&self, py: Python<'py>) -> PyResult> { let c_schema = FFI_ArrowSchema::try_from(self).map_err(to_py_err)?; - let c_schema_ptr = &c_schema as *const FFI_ArrowSchema; - let dtype = - field_class(py)?.call_method1("_import_from_c", (c_schema_ptr as Py_uintptr_t,))?; - Ok(dtype) + field_class(py)?.call_method1("_import_from_c", (&raw const c_schema as Py_uintptr_t,)) } } @@ -234,36 +215,27 @@ impl FromPyArrow for Schema { // method, so prefer it over _export_to_c. // See https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html if value.hasattr("__arrow_c_schema__")? { - let capsule = value.getattr("__arrow_c_schema__")?.call0()?; - let capsule = capsule.cast::()?; - validate_pycapsule(capsule, "arrow_schema")?; + let capsule = value.call_method0("__arrow_c_schema__")?.extract()?; + validate_pycapsule(&capsule, "arrow_schema")?; let schema_ptr = capsule .pointer_checked(Some(ARROW_SCHEMA_CAPSULE_NAME))? .cast::(); - unsafe { - let schema = Schema::try_from(schema_ptr.as_ref()).map_err(to_py_err)?; - return Ok(schema); - } + return unsafe { Schema::try_from(schema_ptr.as_ref()) }.map_err(to_py_err); } validate_class(schema_class(value.py())?, value)?; - let c_schema = FFI_ArrowSchema::empty(); - let c_schema_ptr = &c_schema as *const FFI_ArrowSchema; - value.call_method1("_export_to_c", (c_schema_ptr as Py_uintptr_t,))?; - let schema = Schema::try_from(&c_schema).map_err(to_py_err)?; - Ok(schema) + let mut c_schema = FFI_ArrowSchema::empty(); + value.call_method1("_export_to_c", (&raw mut c_schema as Py_uintptr_t,))?; + Schema::try_from(&c_schema).map_err(to_py_err) } } impl ToPyArrow for Schema { fn to_pyarrow<'py>(&self, py: Python<'py>) -> PyResult> { let c_schema = FFI_ArrowSchema::try_from(self).map_err(to_py_err)?; - let c_schema_ptr = &c_schema as *const FFI_ArrowSchema; - let schema = - schema_class(py)?.call_method1("_import_from_c", (c_schema_ptr as Py_uintptr_t,))?; - Ok(schema) + schema_class(py)?.call_method1("_import_from_c", (&raw const c_schema as Py_uintptr_t,)) } } @@ -273,21 +245,11 @@ impl FromPyArrow for ArrayData { // method, so prefer it over _export_to_c. // See https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html if value.hasattr("__arrow_c_array__")? { - let tuple = value.getattr("__arrow_c_array__")?.call0()?; - - if !tuple.is_instance_of::() { - return Err(PyTypeError::new_err( - "Expected __arrow_c_array__ to return a tuple.", - )); - } - - let schema_capsule = tuple.get_item(0)?; - let schema_capsule = schema_capsule.cast::()?; - let array_capsule = tuple.get_item(1)?; - let array_capsule = array_capsule.cast::()?; + let (schema_capsule, array_capsule) = + value.call_method0("__arrow_c_array__")?.extract()?; - validate_pycapsule(schema_capsule, "arrow_schema")?; - validate_pycapsule(array_capsule, "arrow_array")?; + validate_pycapsule(&schema_capsule, "arrow_schema")?; + validate_pycapsule(&array_capsule, "arrow_array")?; let schema_ptr = schema_capsule .pointer_checked(Some(ARROW_SCHEMA_CAPSULE_NAME))? @@ -315,8 +277,8 @@ impl FromPyArrow for ArrayData { value.call_method1( "_export_to_c", ( - addr_of_mut!(array) as Py_uintptr_t, - addr_of_mut!(schema) as Py_uintptr_t, + &raw mut array as Py_uintptr_t, + &raw mut schema as Py_uintptr_t, ), )?; @@ -328,15 +290,13 @@ impl ToPyArrow for ArrayData { fn to_pyarrow<'py>(&self, py: Python<'py>) -> PyResult> { let array = FFI_ArrowArray::new(self); let schema = FFI_ArrowSchema::try_from(self.data_type()).map_err(to_py_err)?; - - let array = array_class(py)?.call_method1( + array_class(py)?.call_method1( "_import_from_c", ( - addr_of!(array) as Py_uintptr_t, - addr_of!(schema) as Py_uintptr_t, + &raw const array as Py_uintptr_t, + &raw const schema as Py_uintptr_t, ), - )?; - Ok(array) + ) } } @@ -364,21 +324,11 @@ impl FromPyArrow for RecordBatch { // See https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html if value.hasattr("__arrow_c_array__")? { - let tuple = value.getattr("__arrow_c_array__")?.call0()?; + let (schema_capsule, array_capsule) = + value.call_method0("__arrow_c_array__")?.extract()?; - if !tuple.is_instance_of::() { - return Err(PyTypeError::new_err( - "Expected __arrow_c_array__ to return a tuple.", - )); - } - - let schema_capsule = tuple.get_item(0)?; - let schema_capsule = schema_capsule.cast::()?; - let array_capsule = tuple.get_item(1)?; - let array_capsule = array_capsule.cast::()?; - - validate_pycapsule(schema_capsule, "arrow_schema")?; - validate_pycapsule(array_capsule, "arrow_array")?; + validate_pycapsule(&schema_capsule, "arrow_schema")?; + validate_pycapsule(&array_capsule, "arrow_array")?; let schema_ptr = schema_capsule .pointer_checked(Some(ARROW_SCHEMA_CAPSULE_NAME))? @@ -455,9 +405,9 @@ impl FromPyArrow for ArrowArrayStreamReader { // method, so prefer it over _export_to_c. // See https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html if value.hasattr("__arrow_c_stream__")? { - let capsule = value.getattr("__arrow_c_stream__")?.call0()?; - let capsule = capsule.cast::()?; - validate_pycapsule(capsule, "arrow_array_stream")?; + let capsule = value.call_method0("__arrow_c_stream__")?.extract()?; + + validate_pycapsule(&capsule, "arrow_array_stream")?; let stream = unsafe { FFI_ArrowArrayStream::from_raw( @@ -476,20 +426,17 @@ impl FromPyArrow for ArrowArrayStreamReader { validate_class(record_batch_reader_class(value.py())?, value)?; - // prepare a pointer to receive the stream struct + // prepare the stream struct to receive the content let mut stream = FFI_ArrowArrayStream::empty(); - let stream_ptr = &mut stream as *mut FFI_ArrowArrayStream; // make the conversion through PyArrow's private API // this changes the pointer's memory and is thus unsafe. // In particular, `_export_to_c` can go out of bounds - let args = PyTuple::new(value.py(), [stream_ptr as Py_uintptr_t])?; + let args = PyTuple::new(value.py(), [&raw mut stream as Py_uintptr_t])?; value.call_method1("_export_to_c", args)?; - let stream_reader = ArrowArrayStreamReader::try_new(stream) - .map_err(|err| PyValueError::new_err(err.to_string()))?; - - Ok(stream_reader) + ArrowArrayStreamReader::try_new(stream) + .map_err(|err| PyValueError::new_err(err.to_string())) } } @@ -498,13 +445,9 @@ impl IntoPyArrow for Box { // We can't implement `ToPyArrow` for `T: RecordBatchReader + Send` because // there is already a blanket implementation for `T: ToPyArrow`. fn into_pyarrow<'py>(self, py: Python<'py>) -> PyResult> { - let mut stream = FFI_ArrowArrayStream::new(self); - - let stream_ptr = (&mut stream) as *mut FFI_ArrowArrayStream; - let reader = record_batch_reader_class(py)? - .call_method1("_import_from_c", (stream_ptr as Py_uintptr_t,))?; - - Ok(reader) + let stream = FFI_ArrowArrayStream::new(self); + record_batch_reader_class(py)? + .call_method1("_import_from_c", (&raw const stream as Py_uintptr_t,)) } } @@ -588,7 +531,7 @@ impl FromPyArrow for Table { fn from_pyarrow_bound(ob: &Bound) -> PyResult { let reader: Box = Box::new(ArrowArrayStreamReader::from_pyarrow_bound(ob)?); - Self::try_from(reader).map_err(|err| PyErr::new::(err.to_string())) + Self::try_from(reader).map_err(|err| PyValueError::new_err(err.to_string())) } } @@ -601,9 +544,7 @@ impl IntoPyArrow for Table { let kwargs = PyDict::new(py); kwargs.set_item("schema", py_schema)?; - let reader = table_class(py)?.call_method("from_batches", (py_batches,), Some(&kwargs))?; - - Ok(reader) + table_class(py)?.call_method("from_batches", (py_batches,), Some(&kwargs)) } } @@ -664,7 +605,7 @@ impl<'py, T: IntoPyArrow> IntoPyObject<'py> for PyArrowType { type Error = PyErr; - fn into_pyobject(self, py: Python<'py>) -> Result { + fn into_pyobject(self, py: Python<'py>) -> PyResult { self.0.into_pyarrow(py) } } From 937a05e1e600e0c2ad0896d879ddbfba328a1ab4 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 31 Mar 2026 15:46:26 -0400 Subject: [PATCH 16/17] [arrow-pyarrow]: restore __arrow_c_array__ tuple error --- arrow-pyarrow/src/lib.rs | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/arrow-pyarrow/src/lib.rs b/arrow-pyarrow/src/lib.rs index 95f1d38fddf3..d8f584e396d3 100644 --- a/arrow-pyarrow/src/lib.rs +++ b/arrow-pyarrow/src/lib.rs @@ -149,6 +149,24 @@ fn validate_pycapsule(capsule: &Bound, name: &str) -> PyResult<()> { Ok(()) } +fn extract_arrow_c_array_capsules<'py>( + value: &Bound<'py, PyAny>, +) -> PyResult<(Bound<'py, PyCapsule>, Bound<'py, PyCapsule>)> { + let tuple = value.call_method0("__arrow_c_array__")?; + + if !tuple.is_instance_of::() { + return Err(PyTypeError::new_err( + "Expected __arrow_c_array__ to return a tuple of (schema, array) capsules.", + )); + } + + tuple.extract().map_err(|_| { + PyTypeError::new_err( + "Expected __arrow_c_array__ to return a tuple of (schema, array) capsules.", + ) + }) +} + impl FromPyArrow for DataType { fn from_pyarrow_bound(value: &Bound) -> PyResult { // Newer versions of PyArrow as well as other libraries with Arrow data implement this @@ -245,8 +263,7 @@ impl FromPyArrow for ArrayData { // method, so prefer it over _export_to_c. // See https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html if value.hasattr("__arrow_c_array__")? { - let (schema_capsule, array_capsule) = - value.call_method0("__arrow_c_array__")?.extract()?; + let (schema_capsule, array_capsule) = extract_arrow_c_array_capsules(value)?; validate_pycapsule(&schema_capsule, "arrow_schema")?; validate_pycapsule(&array_capsule, "arrow_array")?; @@ -324,8 +341,7 @@ impl FromPyArrow for RecordBatch { // See https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html if value.hasattr("__arrow_c_array__")? { - let (schema_capsule, array_capsule) = - value.call_method0("__arrow_c_array__")?.extract()?; + let (schema_capsule, array_capsule) = extract_arrow_c_array_capsules(value)?; validate_pycapsule(&schema_capsule, "arrow_schema")?; validate_pycapsule(&array_capsule, "arrow_array")?; From bb0edace5748f2a2b1f9ab20a72c5d7a7a93002e Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 31 Mar 2026 16:40:13 -0400 Subject: [PATCH 17/17] [arrow-pyarrow]: test __arrow_c_array__ tuple error --- arrow-pyarrow-testing/tests/pyarrow.rs | 31 ++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/arrow-pyarrow-testing/tests/pyarrow.rs b/arrow-pyarrow-testing/tests/pyarrow.rs index 4ca661b104d2..6f3606478c72 100644 --- a/arrow-pyarrow-testing/tests/pyarrow.rs +++ b/arrow-pyarrow-testing/tests/pyarrow.rs @@ -42,7 +42,10 @@ use arrow_array::{ Array, ArrayRef, BinaryViewArray, Int32Array, RecordBatch, StringArray, StringViewArray, }; use arrow_pyarrow::{FromPyArrow, ToPyArrow}; +use pyo3::exceptions::PyTypeError; +use pyo3::types::{PyAnyMethods, PyModule}; use pyo3::Python; +use std::ffi::CString; use std::sync::Arc; #[test] @@ -94,6 +97,34 @@ fn test_to_pyarrow_byte_view() { } } +#[test] +fn test_from_pyarrow_non_tuple() { + Python::initialize(); + + Python::attach(|py| { + let code = CString::new( + r#" +class NotATuple: + def __arrow_c_array__(self): + return 1 + +value = NotATuple() +"#, + ) + .unwrap(); + + let module = PyModule::from_code(py, code.as_c_str(), c"test.py", c"test_module").unwrap(); + let value = module.getattr("value").unwrap(); + + let err = RecordBatch::from_pyarrow_bound(&value).unwrap_err(); + assert!(err.is_instance_of::(py)); + assert_eq!( + err.to_string(), + "TypeError: Expected __arrow_c_array__ to return a tuple of (schema, array) capsules." + ); + }); +} + fn binary_view_column(num_variadic_buffers: usize) -> BinaryViewArray { let long_scalar = b"but soft what light through yonder window breaks".as_slice(); let mut builder = BinaryViewBuilder::new().with_fixed_block_size(long_scalar.len() as u32);