diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index 8bf2ec3ccf79..1aa069bd4b22 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -44,7 +44,7 @@ jobs: github.event_name == 'pull_request_target' && (github.event.action == 'opened' || github.event.action == 'synchronize') - uses: actions/labeler@v6.0.1 + uses: actions/labeler@v6.1.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} configuration-path: .github/workflows/dev_pr/labeler.yml diff --git a/CHANGELOG-old.md b/CHANGELOG-old.md index c400a53c03db..0fab24890847 100644 --- a/CHANGELOG-old.md +++ b/CHANGELOG-old.md @@ -21,6 +21,181 @@ # Changelog +## [58.2.0](https://github.com/apache/arrow-rs/tree/58.2.0) (2026-04-28) + +[Full Changelog](https://github.com/apache/arrow-rs/compare/58.1.0...58.2.0) + +**Implemented enhancements:** + +- Expose ColumnCloseResult on ArrowColumnChunk [\#9774](https://github.com/apache/arrow-rs/issues/9774) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Expose FFI data structures fields [\#9771](https://github.com/apache/arrow-rs/issues/9771) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- short-circuit last predicate in `RowFilter` when `with_limit(N)` is set [\#9765](https://github.com/apache/arrow-rs/issues/9765) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- vectorise dict-index bounds check [\#9747](https://github.com/apache/arrow-rs/issues/9747) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Refactor `RleEncoder::flush_bit_packed_run` [\#9734](https://github.com/apache/arrow-rs/issues/9734) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Add benchmark for cast from/to decimals [\#9728](https://github.com/apache/arrow-rs/issues/9728) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Add a security policy for arrow-rs [\#9727](https://github.com/apache/arrow-rs/issues/9727) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] [[arrow-flight](https://github.com/apache/arrow-rs/labels/arrow-flight)] +- Support `FixedSizeList` in arrow-json reader [\#9714](https://github.com/apache/arrow-rs/issues/9714) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[Variant\] Add `VariantArrayBuilder::append_nulls` API [\#9684](https://github.com/apache/arrow-rs/issues/9684) +- \[Json\] RunEndEncoded decoder optimization [\#9645](https://github.com/apache/arrow-rs/issues/9645) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[Variant\] `variant_get(..., List<_>)` non-Struct types support [\#9615](https://github.com/apache/arrow-rs/issues/9615) +- \[Variant\] Add unshredded `Struct` fast-path for `variant_get(..., Struct)` [\#9596](https://github.com/apache/arrow-rs/issues/9596) +- Allow setting custom line terminator for CSV writer [\#9571](https://github.com/apache/arrow-rs/issues/9571) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[Variant\] Align cast logic for `variant_get` to cast kernel for numeric/bool types [\#9564](https://github.com/apache/arrow-rs/issues/9564) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- ci: use ubuntu-slim where applicable [\#9536](https://github.com/apache/arrow-rs/issues/9536) +- Publicly export `arrow_string::Predicate` and its methods? [\#9480](https://github.com/apache/arrow-rs/issues/9480) +- Don't create CompressionContext when no compression is selected \[IPC\] [\#9463](https://github.com/apache/arrow-rs/issues/9463) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Parquet: Raw level buffering causes unbounded memory growth for sparse columns [\#9446](https://github.com/apache/arrow-rs/issues/9446) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Parallel Parquet Reading [\#9381](https://github.com/apache/arrow-rs/issues/9381) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] + +**Fixed bugs:** + +- \[Variant\] `unshred_variant` panics on malformed bytes despite returning `Result` [\#9740](https://github.com/apache/arrow-rs/issues/9740) +- RecordBatch::normalize\(\) does not propagate top level null bitmap into the results [\#9732](https://github.com/apache/arrow-rs/issues/9732) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Incorrect accounting in `DictEncoder::estimated_memory_size` [\#9719](https://github.com/apache/arrow-rs/issues/9719) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- arrow-ipc writer does not comply with spec for empty variable-size arrays [\#9716](https://github.com/apache/arrow-rs/issues/9716) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Panic when reading corrupt parquet file with truncated data instead of ParquetError [\#9705](https://github.com/apache/arrow-rs/issues/9705) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- NOTICE.txt is inaccurate [\#9703](https://github.com/apache/arrow-rs/issues/9703) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Unnecessary dependency on regex crate [\#9672](https://github.com/apache/arrow-rs/issues/9672) +- \[arrow-avro\] Avro reader produces incorrect results when reader schema and writer schema differ [\#9655](https://github.com/apache/arrow-rs/issues/9655) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- parquet docs are broken on docs.rs [\#9649](https://github.com/apache/arrow-rs/issues/9649) +- \[Parquet\] ArrowWriter with CDC panics on nested ListArrays [\#9637](https://github.com/apache/arrow-rs/issues/9637) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] [[arrow-flight](https://github.com/apache/arrow-rs/labels/arrow-flight)] +- Use release KEYS file for verification instead of dev KEYS [\#9603](https://github.com/apache/arrow-rs/issues/9603) +- IPC reader: handling of dictionaries with only null values [\#9595](https://github.com/apache/arrow-rs/issues/9595) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Parquet RleDecoder::get\_batch\_with\_dict panics on oob dictionary indices [\#9434](https://github.com/apache/arrow-rs/issues/9434) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] + +**Documentation updates:** + +- docs\(variant\): link VariantArray doc to official Parquet Variant extension type [\#9779](https://github.com/apache/arrow-rs/pull/9779) ([mcharrel](https://github.com/mcharrel)) +- Document Security Policy [\#9730](https://github.com/apache/arrow-rs/pull/9730) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] [[arrow-flight](https://github.com/apache/arrow-rs/labels/arrow-flight)] ([alamb](https://github.com/alamb)) +- Docs: add example of how to read parquet row groups in parallel [\#9396](https://github.com/apache/arrow-rs/pull/9396) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([alamb](https://github.com/alamb)) + +**Performance improvements:** + +- parquet: avoid decode and heap allocation on terminal skip in DeltaBitPackDecoder [\#9784](https://github.com/apache/arrow-rs/issues/9784) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- parquet: O\(1\) skip for bw=0 miniblocks in DeltaBitPackDecoder [\#9783](https://github.com/apache/arrow-rs/issues/9783) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Remove per-message flush overhead in Arrow IPC writer [\#9762](https://github.com/apache/arrow-rs/issues/9762) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Support `GenericListViewArray::new_unchecked` and refactor ListView json decoder [\#9646](https://github.com/apache/arrow-rs/issues/9646) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Support nested REE in arrow-ord `partition` function [\#9640](https://github.com/apache/arrow-rs/issues/9640) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[Parquet\] Remove the BIT\_PACKED encoder [\#9635](https://github.com/apache/arrow-rs/issues/9635) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Pre-reserve output capacity in ByteView/ByteArray dictionary decoding [\#9587](https://github.com/apache/arrow-rs/issues/9587) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Fuse RLE decoding and view gathering for StringView dictionary decoding [\#9582](https://github.com/apache/arrow-rs/issues/9582) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Use branchless index clamping and add get\_batch\_direct to RleDecoder [\#9581](https://github.com/apache/arrow-rs/issues/9581) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Reduce per-byte overhead in VLQ integer decoding [\#9580](https://github.com/apache/arrow-rs/issues/9580) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- feat\(parquet\): batch RLE runs in level encoder via scan-ahead [\#9830](https://github.com/apache/arrow-rs/pull/9830) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- fix: lazy-init zstd compression contexts to avoid unnecessary FFI calls [\#9808](https://github.com/apache/arrow-rs/pull/9808) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mbutrovich](https://github.com/mbutrovich)) +- parquet: O\(1\) skip for bw=0 miniblocks in DeltaBitPackDecoder [\#9786](https://github.com/apache/arrow-rs/pull/9786) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([sahuagin](https://github.com/sahuagin)) +- chore: add benchmark for row filters with LIMIT short-circuit [\#9767](https://github.com/apache/arrow-rs/pull/9767) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([haohuaijin](https://github.com/haohuaijin)) +- Push `LIMIT` / `OFFSET` into the last `RowFilter` predicate and skip unused row groups [\#9766](https://github.com/apache/arrow-rs/pull/9766) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([haohuaijin](https://github.com/haohuaijin)) +- feat\(ipc\): Remove per-message flush in IPC writer hot path [\#9763](https://github.com/apache/arrow-rs/pull/9763) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pchintar](https://github.com/pchintar)) +- perf\(parquet\): Defer fixed length byte array buffer alloc and skip zero-batch init [\#9756](https://github.com/apache/arrow-rs/pull/9756) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([lyang24](https://github.com/lyang24)) +- feat\(parquet\): batch consecutive null/empty rows in `write_list` [\#9752](https://github.com/apache/arrow-rs/pull/9752) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- Remove `len` field from buffer builder [\#9750](https://github.com/apache/arrow-rs/pull/9750) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([cetra3](https://github.com/cetra3)) +- perf\(parquet\): Vectorize dict-index bounds check in RleDecoder::get\_batch\_with\_dict \(up to -7.9%\) [\#9746](https://github.com/apache/arrow-rs/pull/9746) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) +- feat\(parquet\): precompute `offset_index_disabled` at build-time [\#9724](https://github.com/apache/arrow-rs/pull/9724) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- \[Parquet\] Improve dictionary decoder by unrolling loops [\#9662](https://github.com/apache/arrow-rs/pull/9662) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) +- \[Json\] Use `partition` and `take` in RunEndEncoded decoder [\#9658](https://github.com/apache/arrow-rs/pull/9658) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- Improve take performance on List arrays [\#9643](https://github.com/apache/arrow-rs/pull/9643) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([AdamGS](https://github.com/AdamGS)) +- \[Json\] Replace `ArrayData` with typed Array construction in json-reader [\#9497](https://github.com/apache/arrow-rs/pull/9497) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- feat\(parquet\): stream-encode definition/repetition levels incrementally [\#9447](https://github.com/apache/arrow-rs/pull/9447) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) + +**Closed issues:** + +- Incorrect buffer skipping for V4 Union types in IPC `skip_field` [\#9828](https://github.com/apache/arrow-rs/issues/9828) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Replace wildcard match in `skip_field` with explicit `DataType` handling [\#9821](https://github.com/apache/arrow-rs/issues/9821) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Column projection misalignment for ListView / LargeListView in IPC reader [\#9805](https://github.com/apache/arrow-rs/issues/9805) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Avoid panic on malformed compressed buffer prefix in IPC [\#9801](https://github.com/apache/arrow-rs/issues/9801) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- DeltaByteArrayDecoder panics on invalid prefix lengths [\#9796](https://github.com/apache/arrow-rs/issues/9796) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Use NullBufferBuilder when reading json [\#9781](https://github.com/apache/arrow-rs/issues/9781) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Perfectly shredded arrays with top-level null values loss nullability when `typed_value` is extracted [\#9701](https://github.com/apache/arrow-rs/issues/9701) +- \[Parquet Metadata\] API to determine page-index presence separately from page-index load [\#9693](https://github.com/apache/arrow-rs/issues/9693) +- Union cast is incorrect for duplicate field names [\#9664](https://github.com/apache/arrow-rs/issues/9664) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- List and ListView are missing `take` benchmarks [\#9627](https://github.com/apache/arrow-rs/issues/9627) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Support RunEndEncoded arrays in comparison kernels \(eq, lt, etc.\) [\#9620](https://github.com/apache/arrow-rs/issues/9620) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- variant\_get should follow JSONpath semantics [\#9606](https://github.com/apache/arrow-rs/issues/9606) +- GenericByteViewArray: support finding total length of all strings [\#9435](https://github.com/apache/arrow-rs/issues/9435) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] + +**Merged pull requests:** + +- support length\(\) on Run-end encoding arrays [\#9838](https://github.com/apache/arrow-rs/pull/9838) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Rich-T-kid](https://github.com/Rich-T-kid)) +- fix\(ipc\): correct skip\_field handling for V4 Union [\#9829](https://github.com/apache/arrow-rs/pull/9829) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pchintar](https://github.com/pchintar)) +- fix\(ipc\): replace wildcard in skip\_field with explicit DataType handling [\#9822](https://github.com/apache/arrow-rs/pull/9822) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pchintar](https://github.com/pchintar)) +- Prevent buffer builder length overflow in `MutableBuffer::extend_zeros` [\#9820](https://github.com/apache/arrow-rs/pull/9820) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Prevent repeat slice length overflow [\#9819](https://github.com/apache/arrow-rs/pull/9819) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Prevent BitChunks length overflow [\#9818](https://github.com/apache/arrow-rs/pull/9818) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Prevent Rows row index overflow [\#9817](https://github.com/apache/arrow-rs/pull/9817) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Prevent ArrayData validation length overflow [\#9816](https://github.com/apache/arrow-rs/pull/9816) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- \[Json\] Remove arrow-data dependency from arrow-json [\#9812](https://github.com/apache/arrow-rs/pull/9812) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- Replace `BooleanBufferBuilder` with `NullBufferBuilder` in arrow-json if applicable [\#9811](https://github.com/apache/arrow-rs/pull/9811) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- refactor\(ipc\): derive Default for CompressionContext [\#9809](https://github.com/apache/arrow-rs/pull/9809) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mbutrovich](https://github.com/mbutrovich)) +- fix\(ipc\): reader misalignment when skipping ListView / LargeListView columns [\#9806](https://github.com/apache/arrow-rs/pull/9806) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pchintar](https://github.com/pchintar)) +- fix\(ipc\): Avoid panic on malformed compressed buffer prefix [\#9802](https://github.com/apache/arrow-rs/pull/9802) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pchintar](https://github.com/pchintar)) +- parquet: fix panic in DeltaByteArrayDecoder on invalid prefix lengths [\#9797](https://github.com/apache/arrow-rs/pull/9797) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([pchintar](https://github.com/pchintar)) +- feat\(parquet\): fuse level encoding with counting and histogram updates [\#9795](https://github.com/apache/arrow-rs/pull/9795) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- Expose ColumnCloseResult on ArrowColumnChunk [\#9773](https://github.com/apache/arrow-rs/pull/9773) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([leoyvens](https://github.com/leoyvens)) +- feat: make FFI structs fields `pub` [\#9772](https://github.com/apache/arrow-rs/pull/9772) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([ashdnazg](https://github.com/ashdnazg)) +- chore: Refine the error message for List to non List cast [\#9757](https://github.com/apache/arrow-rs/pull/9757) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([comphead](https://github.com/comphead)) +- refactor\(parquet\): replace magic `8` literals with named constants [\#9751](https://github.com/apache/arrow-rs/pull/9751) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- feat\(ipc\): add with\_skip\_validation to StreamDecoder [\#9749](https://github.com/apache/arrow-rs/pull/9749) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pantShrey](https://github.com/pantShrey)) +- remove panics in unshred variant [\#9741](https://github.com/apache/arrow-rs/pull/9741) ([friendlymatthew](https://github.com/friendlymatthew)) +- Add benchmark for ListView interleave [\#9738](https://github.com/apache/arrow-rs/pull/9738) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([vegarsti](https://github.com/vegarsti)) +- arrow-arith: fix 'occured' -\> 'occurred' in arity.rs comments [\#9736](https://github.com/apache/arrow-rs/pull/9736) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([SAY-5](https://github.com/SAY-5)) +- Refactor `RleEncoder::flush_bit_packed_run` to make flow clearer [\#9735](https://github.com/apache/arrow-rs/pull/9735) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://github.com/etseidl)) +- Fix RecordBatch::normalize\(\) null bitmap bug and add StructArray::flatten\(\) [\#9733](https://github.com/apache/arrow-rs/pull/9733) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([sqd](https://github.com/sqd)) +- Add benchmark for cast from/to decimals [\#9729](https://github.com/apache/arrow-rs/pull/9729) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([klion26](https://github.com/klion26)) +- refactor\(arrow-avro\): use `Decoder::flush_block` in async reader [\#9726](https://github.com/apache/arrow-rs/pull/9726) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mzabaluev](https://github.com/mzabaluev)) +- fix: ParquetError when reading corrupt parquet file with truncated data instead of Panic [\#9725](https://github.com/apache/arrow-rs/pull/9725) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([xuzifu666](https://github.com/xuzifu666)) +- feat\(parquet\): add wide-schema writer overhead benchmark [\#9723](https://github.com/apache/arrow-rs/pull/9723) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- fix: correct accounting in `DictEncoder::estimated_memory_size`, `Interner::estimated_memory_size` [\#9720](https://github.com/apache/arrow-rs/pull/9720) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([mzabaluev](https://github.com/mzabaluev)) +- arrow-ipc: Write 0 offset buffer for length-0 variable-size arrays [\#9717](https://github.com/apache/arrow-rs/pull/9717) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([atwam](https://github.com/atwam)) +- \[Json\] Support `FixedSizeList` in json decoder [\#9715](https://github.com/apache/arrow-rs/pull/9715) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- chore\(deps\): bump actions/upload-pages-artifact from 4 to 5 [\#9713](https://github.com/apache/arrow-rs/pull/9713) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Fix clippy warning in fixed\_size\_binary\_array.rs [\#9712](https://github.com/apache/arrow-rs/pull/9712) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([AdamGS](https://github.com/AdamGS)) +- feat: add `has_non_empty_nulls` helper function in `OffsetBuffer` [\#9711](https://github.com/apache/arrow-rs/pull/9711) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) +- chore\(deps\): bump pytest from 7.2.0 to 9.0.3 in /parquet/pytest [\#9706](https://github.com/apache/arrow-rs/pull/9706) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([dependabot[bot]](https://github.com/apps/dependabot)) +- Fedora license audit [\#9704](https://github.com/apache/arrow-rs/pull/9704) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([michel-slm](https://github.com/michel-slm)) +- \[Variant\] Take top-level nulls into consideration when extracting perfectly shredded children [\#9702](https://github.com/apache/arrow-rs/pull/9702) ([AdamGS](https://github.com/AdamGS)) +- feat\(parquet\): add `push_decoder` benchmark for `PushBuffers` overhead [\#9696](https://github.com/apache/arrow-rs/pull/9696) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- Add mutable bitwise operations to `BooleanArray` and `NullBuffer::union_many` [\#9692](https://github.com/apache/arrow-rs/pull/9692) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mbutrovich](https://github.com/mbutrovich)) +- chore\(deps\): update hashbrown requirement from 0.16.0 to 0.17.0 [\#9691](https://github.com/apache/arrow-rs/pull/9691) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([dependabot[bot]](https://github.com/apps/dependabot)) +- chore\(deps\): bump actions/github-script from 8 to 9 [\#9690](https://github.com/apache/arrow-rs/pull/9690) ([dependabot[bot]](https://github.com/apps/dependabot)) +- minor: Re-enable CDC bench [\#9686](https://github.com/apache/arrow-rs/pull/9686) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://github.com/etseidl)) +- \[Variant\] Add `VariantArrayBuilder::append_nulls` API [\#9685](https://github.com/apache/arrow-rs/pull/9685) ([sdf-jkl](https://github.com/sdf-jkl)) +- feat\(parquet\): add struct-column writer benchmarks [\#9679](https://github.com/apache/arrow-rs/pull/9679) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- \[Arrow\] Add API to check if `Field` has a valid `ExtensionType` [\#9677](https://github.com/apache/arrow-rs/pull/9677) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([sdf-jkl](https://github.com/sdf-jkl)) +- \[Variant\] `variant_get` should follow JSONPath semantics for Field path element [\#9676](https://github.com/apache/arrow-rs/pull/9676) ([sdf-jkl](https://github.com/sdf-jkl)) +- ParquetMetaDataPushDecoder API to clear all buffered ranges [\#9673](https://github.com/apache/arrow-rs/pull/9673) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([nathanb9](https://github.com/nathanb9)) +- Fix union cast incorrectness for duplicate field names [\#9666](https://github.com/apache/arrow-rs/pull/9666) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([friendlymatthew](https://github.com/friendlymatthew)) +- chore: re-export `MAX_INLINE_VIEW_LEN` from `arrow_data` [\#9665](https://github.com/apache/arrow-rs/pull/9665) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) +- No longer allow BIT\_PACKED level encoding in Parquet writer [\#9656](https://github.com/apache/arrow-rs/pull/9656) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://github.com/etseidl)) +- feat\(parquet\): add sparse-column writer benchmarks [\#9654](https://github.com/apache/arrow-rs/pull/9654) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- Support `GenericListViewArray::new_unchecked` and refactor `ListView` json decoder [\#9648](https://github.com/apache/arrow-rs/pull/9648) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- \[Json\] Add json reader benchmarks for ListView [\#9647](https://github.com/apache/arrow-rs/pull/9647) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- fix\(parquet\): fix CDC panic on nested ListArrays with null entries [\#9644](https://github.com/apache/arrow-rs/pull/9644) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([kszucs](https://github.com/kszucs)) +- Add a test for reading nested REE data in json [\#9634](https://github.com/apache/arrow-rs/pull/9634) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- \[Variant\] Fix `variant_get` to return `List` instead of `List` [\#9631](https://github.com/apache/arrow-rs/pull/9631) ([liamzwbao](https://github.com/liamzwbao)) +- ci: use ubuntu-slim runner for lightweight CI jobs [\#9630](https://github.com/apache/arrow-rs/pull/9630) ([CuteChuanChuan](https://github.com/CuteChuanChuan)) +- Add bloom filter folding to automatically size SBBF filters [\#9628](https://github.com/apache/arrow-rs/pull/9628) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([adriangb](https://github.com/adriangb)) +- Add List and ListView take benchmarks [\#9626](https://github.com/apache/arrow-rs/pull/9626) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([AdamGS](https://github.com/AdamGS)) +- ParquetPushDecoder API to clear all buffered ranges [\#9624](https://github.com/apache/arrow-rs/pull/9624) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([nathanb9](https://github.com/nathanb9)) +- fix: handle missing dictionary batch for null-only columns in IPC reader [\#9623](https://github.com/apache/arrow-rs/pull/9623) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([joaquinhuigomez](https://github.com/joaquinhuigomez)) +- Fix `MutableBuffer::clear` [\#9622](https://github.com/apache/arrow-rs/pull/9622) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Rafferty97](https://github.com/Rafferty97)) +- feat\[arrow-ord\]: suppport REE comparisons [\#9621](https://github.com/apache/arrow-rs/pull/9621) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([asubiotto](https://github.com/asubiotto)) +- chore\(deps\): update sha2 requirement from 0.10 to 0.11 [\#9618](https://github.com/apache/arrow-rs/pull/9618) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([dependabot[bot]](https://github.com/apps/dependabot)) +- Expose option to set line terminator for CSV writer [\#9617](https://github.com/apache/arrow-rs/pull/9617) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([svranesevic](https://github.com/svranesevic)) +- \[Json\] Add json reader benchmarks for Map and REE [\#9616](https://github.com/apache/arrow-rs/pull/9616) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- deps: fix `object_store` breakage for 0.13.2 [\#9612](https://github.com/apache/arrow-rs/pull/9612) ([mzabaluev-flarion](https://github.com/mzabaluev-flarion)) +- \[Variant\] Support Binary/LargeBinary children [\#9610](https://github.com/apache/arrow-rs/pull/9610) ([AdamGS](https://github.com/AdamGS)) +- fix: use writer types in Skipper for resolved named record types [\#9605](https://github.com/apache/arrow-rs/pull/9605) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([ariel-miculas](https://github.com/ariel-miculas)) +- feat\(parquet\): derive `PartialEq` and `Eq` for `CdcOptions` [\#9602](https://github.com/apache/arrow-rs/pull/9602) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([kszucs](https://github.com/kszucs)) +- Add `finish_preserve_values` to `ArrayBuilder` trait [\#9601](https://github.com/apache/arrow-rs/pull/9601) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([adamreichold](https://github.com/adamreichold)) +- \[Variant\] extend shredded null handling for arrays [\#9599](https://github.com/apache/arrow-rs/pull/9599) ([sdf-jkl](https://github.com/sdf-jkl)) +- \[Variant\] Add unshredded `Struct` fast-path for `variant_get(..., Struct)` [\#9597](https://github.com/apache/arrow-rs/pull/9597) ([sdf-jkl](https://github.com/sdf-jkl)) +- Pre-reserve output capacity in ByteView/ByteArray dictionary decoding [\#9590](https://github.com/apache/arrow-rs/pull/9590) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) +- \[Variant\] Align cast logic for variant\_get to cast kernel for numeric/bool types [\#9563](https://github.com/apache/arrow-rs/pull/9563) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([klion26](https://github.com/klion26)) +- Add support to cast from `UnionArray` [\#9544](https://github.com/apache/arrow-rs/pull/9544) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([friendlymatthew](https://github.com/friendlymatthew)) +- Support `ListView` codec in arrow-json [\#9503](https://github.com/apache/arrow-rs/pull/9503) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) + + ## [58.1.0](https://github.com/apache/arrow-rs/tree/58.1.0) (2026-03-20) [Full Changelog](https://github.com/apache/arrow-rs/compare/58.0.0...58.1.0) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03c4e1579446..311d5e99a368 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,179 +19,73 @@ # Changelog -## [58.2.0](https://github.com/apache/arrow-rs/tree/58.2.0) (2026-04-28) +## [58.3.0](https://github.com/apache/arrow-rs/tree/58.3.0) (2026-05-07) -[Full Changelog](https://github.com/apache/arrow-rs/compare/58.1.0...58.2.0) +[Full Changelog](https://github.com/apache/arrow-rs/compare/58.2.0...58.3.0) **Implemented enhancements:** -- Expose ColumnCloseResult on ArrowColumnChunk [\#9774](https://github.com/apache/arrow-rs/issues/9774) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- Expose FFI data structures fields [\#9771](https://github.com/apache/arrow-rs/issues/9771) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- short-circuit last predicate in `RowFilter` when `with_limit(N)` is set [\#9765](https://github.com/apache/arrow-rs/issues/9765) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- vectorise dict-index bounds check [\#9747](https://github.com/apache/arrow-rs/issues/9747) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- Refactor `RleEncoder::flush_bit_packed_run` [\#9734](https://github.com/apache/arrow-rs/issues/9734) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- Add benchmark for cast from/to decimals [\#9728](https://github.com/apache/arrow-rs/issues/9728) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Add a security policy for arrow-rs [\#9727](https://github.com/apache/arrow-rs/issues/9727) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] [[arrow-flight](https://github.com/apache/arrow-rs/labels/arrow-flight)] -- Support `FixedSizeList` in arrow-json reader [\#9714](https://github.com/apache/arrow-rs/issues/9714) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- \[Variant\] Add `VariantArrayBuilder::append_nulls` API [\#9684](https://github.com/apache/arrow-rs/issues/9684) -- \[Json\] RunEndEncoded decoder optimization [\#9645](https://github.com/apache/arrow-rs/issues/9645) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- \[Variant\] `variant_get(..., List<_>)` non-Struct types support [\#9615](https://github.com/apache/arrow-rs/issues/9615) -- \[Variant\] Add unshredded `Struct` fast-path for `variant_get(..., Struct)` [\#9596](https://github.com/apache/arrow-rs/issues/9596) -- Allow setting custom line terminator for CSV writer [\#9571](https://github.com/apache/arrow-rs/issues/9571) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- \[Variant\] Align cast logic for `variant_get` to cast kernel for numeric/bool types [\#9564](https://github.com/apache/arrow-rs/issues/9564) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- ci: use ubuntu-slim where applicable [\#9536](https://github.com/apache/arrow-rs/issues/9536) -- Publicly export `arrow_string::Predicate` and its methods? [\#9480](https://github.com/apache/arrow-rs/issues/9480) -- Don't create CompressionContext when no compression is selected \[IPC\] [\#9463](https://github.com/apache/arrow-rs/issues/9463) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Parquet: Raw level buffering causes unbounded memory growth for sparse columns [\#9446](https://github.com/apache/arrow-rs/issues/9446) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- Parallel Parquet Reading [\#9381](https://github.com/apache/arrow-rs/issues/9381) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Add `DatePart::from_str` API [\#9930](https://github.com/apache/arrow-rs/issues/9930) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- should use DictionaryArray::with\_values instead of try\_new on the dictionary fast path [\#9889](https://github.com/apache/arrow-rs/issues/9889) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[arrow-string\] add concat\_elements for BinaryViewArray and FixedSizeBinary [\#9875](https://github.com/apache/arrow-rs/issues/9875) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Expose eq ignore ascii case from arrow-string [\#9870](https://github.com/apache/arrow-rs/issues/9870) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Configurable data page v2 compression threshold [\#9827](https://github.com/apache/arrow-rs/issues/9827) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] **Fixed bugs:** -- \[Variant\] `unshred_variant` panics on malformed bytes despite returning `Result` [\#9740](https://github.com/apache/arrow-rs/issues/9740) -- RecordBatch::normalize\(\) does not propagate top level null bitmap into the results [\#9732](https://github.com/apache/arrow-rs/issues/9732) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Incorrect accounting in `DictEncoder::estimated_memory_size` [\#9719](https://github.com/apache/arrow-rs/issues/9719) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- arrow-ipc writer does not comply with spec for empty variable-size arrays [\#9716](https://github.com/apache/arrow-rs/issues/9716) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Panic when reading corrupt parquet file with truncated data instead of ParquetError [\#9705](https://github.com/apache/arrow-rs/issues/9705) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- NOTICE.txt is inaccurate [\#9703](https://github.com/apache/arrow-rs/issues/9703) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Unnecessary dependency on regex crate [\#9672](https://github.com/apache/arrow-rs/issues/9672) -- \[arrow-avro\] Avro reader produces incorrect results when reader schema and writer schema differ [\#9655](https://github.com/apache/arrow-rs/issues/9655) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- parquet docs are broken on docs.rs [\#9649](https://github.com/apache/arrow-rs/issues/9649) -- \[Parquet\] ArrowWriter with CDC panics on nested ListArrays [\#9637](https://github.com/apache/arrow-rs/issues/9637) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] [[arrow-flight](https://github.com/apache/arrow-rs/labels/arrow-flight)] -- Use release KEYS file for verification instead of dev KEYS [\#9603](https://github.com/apache/arrow-rs/issues/9603) -- IPC reader: handling of dictionaries with only null values [\#9595](https://github.com/apache/arrow-rs/issues/9595) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Parquet RleDecoder::get\_batch\_with\_dict panics on oob dictionary indices [\#9434](https://github.com/apache/arrow-rs/issues/9434) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- \[arrow-cast\] incorrect Time32 -\> Time64 conversion [\#9851](https://github.com/apache/arrow-rs/issues/9851) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Panic when reading malformed compact-Thrift bool fields in Parquet page metadata [\#9839](https://github.com/apache/arrow-rs/issues/9839) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Parquet `DeltaBitPackDecoder::skip` could panic on "non-standard" miniblocks [\#9793](https://github.com/apache/arrow-rs/issues/9793) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] **Documentation updates:** -- docs\(variant\): link VariantArray doc to official Parquet Variant extension type [\#9779](https://github.com/apache/arrow-rs/pull/9779) ([mcharrel](https://github.com/mcharrel)) -- Document Security Policy [\#9730](https://github.com/apache/arrow-rs/pull/9730) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] [[arrow-flight](https://github.com/apache/arrow-rs/labels/arrow-flight)] ([alamb](https://github.com/alamb)) -- Docs: add example of how to read parquet row groups in parallel [\#9396](https://github.com/apache/arrow-rs/pull/9396) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([alamb](https://github.com/alamb)) - -**Performance improvements:** - -- parquet: avoid decode and heap allocation on terminal skip in DeltaBitPackDecoder [\#9784](https://github.com/apache/arrow-rs/issues/9784) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- parquet: O\(1\) skip for bw=0 miniblocks in DeltaBitPackDecoder [\#9783](https://github.com/apache/arrow-rs/issues/9783) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- Remove per-message flush overhead in Arrow IPC writer [\#9762](https://github.com/apache/arrow-rs/issues/9762) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Support `GenericListViewArray::new_unchecked` and refactor ListView json decoder [\#9646](https://github.com/apache/arrow-rs/issues/9646) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Support nested REE in arrow-ord `partition` function [\#9640](https://github.com/apache/arrow-rs/issues/9640) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- \[Parquet\] Remove the BIT\_PACKED encoder [\#9635](https://github.com/apache/arrow-rs/issues/9635) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- Pre-reserve output capacity in ByteView/ByteArray dictionary decoding [\#9587](https://github.com/apache/arrow-rs/issues/9587) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- Fuse RLE decoding and view gathering for StringView dictionary decoding [\#9582](https://github.com/apache/arrow-rs/issues/9582) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- Use branchless index clamping and add get\_batch\_direct to RleDecoder [\#9581](https://github.com/apache/arrow-rs/issues/9581) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- Reduce per-byte overhead in VLQ integer decoding [\#9580](https://github.com/apache/arrow-rs/issues/9580) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- feat\(parquet\): batch RLE runs in level encoder via scan-ahead [\#9830](https://github.com/apache/arrow-rs/pull/9830) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) -- fix: lazy-init zstd compression contexts to avoid unnecessary FFI calls [\#9808](https://github.com/apache/arrow-rs/pull/9808) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mbutrovich](https://github.com/mbutrovich)) -- parquet: O\(1\) skip for bw=0 miniblocks in DeltaBitPackDecoder [\#9786](https://github.com/apache/arrow-rs/pull/9786) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([sahuagin](https://github.com/sahuagin)) -- chore: add benchmark for row filters with LIMIT short-circuit [\#9767](https://github.com/apache/arrow-rs/pull/9767) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([haohuaijin](https://github.com/haohuaijin)) -- Push `LIMIT` / `OFFSET` into the last `RowFilter` predicate and skip unused row groups [\#9766](https://github.com/apache/arrow-rs/pull/9766) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([haohuaijin](https://github.com/haohuaijin)) -- feat\(ipc\): Remove per-message flush in IPC writer hot path [\#9763](https://github.com/apache/arrow-rs/pull/9763) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pchintar](https://github.com/pchintar)) -- perf\(parquet\): Defer fixed length byte array buffer alloc and skip zero-batch init [\#9756](https://github.com/apache/arrow-rs/pull/9756) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([lyang24](https://github.com/lyang24)) -- feat\(parquet\): batch consecutive null/empty rows in `write_list` [\#9752](https://github.com/apache/arrow-rs/pull/9752) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) -- Remove `len` field from buffer builder [\#9750](https://github.com/apache/arrow-rs/pull/9750) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([cetra3](https://github.com/cetra3)) -- perf\(parquet\): Vectorize dict-index bounds check in RleDecoder::get\_batch\_with\_dict \(up to -7.9%\) [\#9746](https://github.com/apache/arrow-rs/pull/9746) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) -- feat\(parquet\): precompute `offset_index_disabled` at build-time [\#9724](https://github.com/apache/arrow-rs/pull/9724) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) -- \[Parquet\] Improve dictionary decoder by unrolling loops [\#9662](https://github.com/apache/arrow-rs/pull/9662) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) -- \[Json\] Use `partition` and `take` in RunEndEncoded decoder [\#9658](https://github.com/apache/arrow-rs/pull/9658) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) -- Improve take performance on List arrays [\#9643](https://github.com/apache/arrow-rs/pull/9643) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([AdamGS](https://github.com/AdamGS)) -- \[Json\] Replace `ArrayData` with typed Array construction in json-reader [\#9497](https://github.com/apache/arrow-rs/pull/9497) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) -- feat\(parquet\): stream-encode definition/repetition levels incrementally [\#9447](https://github.com/apache/arrow-rs/pull/9447) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- docs: Add guidance for AI assisted submissions to CONTRIBUTING.md [\#9892](https://github.com/apache/arrow-rs/pull/9892) ([etseidl](https://github.com/etseidl)) +- Update release schedule on README [\#9881](https://github.com/apache/arrow-rs/pull/9881) ([alamb](https://github.com/alamb)) +- Add more documentation for FixedSizeBinary arrays [\#9866](https://github.com/apache/arrow-rs/pull/9866) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Minor: document why FixedSizeBinary offset is always 0 [\#9861](https://github.com/apache/arrow-rs/pull/9861) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- docs: Update contributing guidelines with benchmark results [\#9782](https://github.com/apache/arrow-rs/pull/9782) ([alamb](https://github.com/alamb)) **Closed issues:** -- Incorrect buffer skipping for V4 Union types in IPC `skip_field` [\#9828](https://github.com/apache/arrow-rs/issues/9828) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Replace wildcard match in `skip_field` with explicit `DataType` handling [\#9821](https://github.com/apache/arrow-rs/issues/9821) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Column projection misalignment for ListView / LargeListView in IPC reader [\#9805](https://github.com/apache/arrow-rs/issues/9805) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Avoid panic on malformed compressed buffer prefix in IPC [\#9801](https://github.com/apache/arrow-rs/issues/9801) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- DeltaByteArrayDecoder panics on invalid prefix lengths [\#9796](https://github.com/apache/arrow-rs/issues/9796) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] -- Use NullBufferBuilder when reading json [\#9781](https://github.com/apache/arrow-rs/issues/9781) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Perfectly shredded arrays with top-level null values loss nullability when `typed_value` is extracted [\#9701](https://github.com/apache/arrow-rs/issues/9701) -- \[Parquet Metadata\] API to determine page-index presence separately from page-index load [\#9693](https://github.com/apache/arrow-rs/issues/9693) -- Union cast is incorrect for duplicate field names [\#9664](https://github.com/apache/arrow-rs/issues/9664) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- List and ListView are missing `take` benchmarks [\#9627](https://github.com/apache/arrow-rs/issues/9627) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- Support RunEndEncoded arrays in comparison kernels \(eq, lt, etc.\) [\#9620](https://github.com/apache/arrow-rs/issues/9620) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] -- variant\_get should follow JSONpath semantics [\#9606](https://github.com/apache/arrow-rs/issues/9606) -- GenericByteViewArray: support finding total length of all strings [\#9435](https://github.com/apache/arrow-rs/issues/9435) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- GenericByteDictionaryBuilder::with\_capacity does not pre-size dedup HashTable [\#9907](https://github.com/apache/arrow-rs/issues/9907) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[arrow-buffer\] Integer overflow in repeat\_slice\_n\_times leads to undefined behavior [\#9904](https://github.com/apache/arrow-rs/issues/9904) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[arrow-buffer\] Integer overflow in BitChunks::new leads to undefined behavior [\#9903](https://github.com/apache/arrow-rs/issues/9903) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[arrow-row\] Integer overflow in Rows::row index handling leads to undefined behavior [\#9901](https://github.com/apache/arrow-rs/issues/9901) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[arrow-data\] Integer overflow in ArrayData validation leads to undefined behavior [\#9900](https://github.com/apache/arrow-rs/issues/9900) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[arrow-data\] Integer overflow in ArrayData::slice leads to undefined behavior [\#9899](https://github.com/apache/arrow-rs/issues/9899) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[arrow-array\] Integer overflow in FixedSizeBinaryArray::value leads to undefined behavior [\#9898](https://github.com/apache/arrow-rs/issues/9898) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- \[arrow-buffer\] Integer overflow in BufferBuilder::reserve leads to undefined behavior [\#9897](https://github.com/apache/arrow-rs/issues/9897) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- arrow-csv: integer overflow panic in Reader::records::flush [\#9885](https://github.com/apache/arrow-rs/issues/9885) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Make an API to help with the pattern of 'replaces the values of the REE array' [\#9854](https://github.com/apache/arrow-rs/issues/9854) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] +- Parquet reader rejects canonical UNKNOWN logical type on BOOLEAN physical columns [\#9844](https://github.com/apache/arrow-rs/issues/9844) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- ColumnIndex length mismatch can cause panic during decoding in Parquet [\#9832](https://github.com/apache/arrow-rs/issues/9832) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] +- Bug converting json to fixed list of zero size [\#9780](https://github.com/apache/arrow-rs/issues/9780) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] **Merged pull requests:** -- support length\(\) on Run-end encoding arrays [\#9838](https://github.com/apache/arrow-rs/pull/9838) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Rich-T-kid](https://github.com/Rich-T-kid)) -- fix\(ipc\): correct skip\_field handling for V4 Union [\#9829](https://github.com/apache/arrow-rs/pull/9829) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pchintar](https://github.com/pchintar)) -- fix\(ipc\): replace wildcard in skip\_field with explicit DataType handling [\#9822](https://github.com/apache/arrow-rs/pull/9822) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pchintar](https://github.com/pchintar)) -- Prevent buffer builder length overflow in `MutableBuffer::extend_zeros` [\#9820](https://github.com/apache/arrow-rs/pull/9820) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Prevent repeat slice length overflow [\#9819](https://github.com/apache/arrow-rs/pull/9819) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Prevent BitChunks length overflow [\#9818](https://github.com/apache/arrow-rs/pull/9818) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Prevent Rows row index overflow [\#9817](https://github.com/apache/arrow-rs/pull/9817) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- Prevent ArrayData validation length overflow [\#9816](https://github.com/apache/arrow-rs/pull/9816) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- \[Json\] Remove arrow-data dependency from arrow-json [\#9812](https://github.com/apache/arrow-rs/pull/9812) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) -- Replace `BooleanBufferBuilder` with `NullBufferBuilder` in arrow-json if applicable [\#9811](https://github.com/apache/arrow-rs/pull/9811) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) -- refactor\(ipc\): derive Default for CompressionContext [\#9809](https://github.com/apache/arrow-rs/pull/9809) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mbutrovich](https://github.com/mbutrovich)) -- fix\(ipc\): reader misalignment when skipping ListView / LargeListView columns [\#9806](https://github.com/apache/arrow-rs/pull/9806) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pchintar](https://github.com/pchintar)) -- fix\(ipc\): Avoid panic on malformed compressed buffer prefix [\#9802](https://github.com/apache/arrow-rs/pull/9802) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pchintar](https://github.com/pchintar)) -- parquet: fix panic in DeltaByteArrayDecoder on invalid prefix lengths [\#9797](https://github.com/apache/arrow-rs/pull/9797) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([pchintar](https://github.com/pchintar)) -- feat\(parquet\): fuse level encoding with counting and histogram updates [\#9795](https://github.com/apache/arrow-rs/pull/9795) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) -- Expose ColumnCloseResult on ArrowColumnChunk [\#9773](https://github.com/apache/arrow-rs/pull/9773) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([leoyvens](https://github.com/leoyvens)) -- feat: make FFI structs fields `pub` [\#9772](https://github.com/apache/arrow-rs/pull/9772) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([ashdnazg](https://github.com/ashdnazg)) -- chore: Refine the error message for List to non List cast [\#9757](https://github.com/apache/arrow-rs/pull/9757) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([comphead](https://github.com/comphead)) -- refactor\(parquet\): replace magic `8` literals with named constants [\#9751](https://github.com/apache/arrow-rs/pull/9751) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) -- feat\(ipc\): add with\_skip\_validation to StreamDecoder [\#9749](https://github.com/apache/arrow-rs/pull/9749) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([pantShrey](https://github.com/pantShrey)) -- remove panics in unshred variant [\#9741](https://github.com/apache/arrow-rs/pull/9741) ([friendlymatthew](https://github.com/friendlymatthew)) -- Add benchmark for ListView interleave [\#9738](https://github.com/apache/arrow-rs/pull/9738) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([vegarsti](https://github.com/vegarsti)) -- arrow-arith: fix 'occured' -\> 'occurred' in arity.rs comments [\#9736](https://github.com/apache/arrow-rs/pull/9736) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([SAY-5](https://github.com/SAY-5)) -- Refactor `RleEncoder::flush_bit_packed_run` to make flow clearer [\#9735](https://github.com/apache/arrow-rs/pull/9735) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://github.com/etseidl)) -- Fix RecordBatch::normalize\(\) null bitmap bug and add StructArray::flatten\(\) [\#9733](https://github.com/apache/arrow-rs/pull/9733) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([sqd](https://github.com/sqd)) -- Add benchmark for cast from/to decimals [\#9729](https://github.com/apache/arrow-rs/pull/9729) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([klion26](https://github.com/klion26)) -- refactor\(arrow-avro\): use `Decoder::flush_block` in async reader [\#9726](https://github.com/apache/arrow-rs/pull/9726) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mzabaluev](https://github.com/mzabaluev)) -- fix: ParquetError when reading corrupt parquet file with truncated data instead of Panic [\#9725](https://github.com/apache/arrow-rs/pull/9725) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([xuzifu666](https://github.com/xuzifu666)) -- feat\(parquet\): add wide-schema writer overhead benchmark [\#9723](https://github.com/apache/arrow-rs/pull/9723) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) -- fix: correct accounting in `DictEncoder::estimated_memory_size`, `Interner::estimated_memory_size` [\#9720](https://github.com/apache/arrow-rs/pull/9720) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([mzabaluev](https://github.com/mzabaluev)) -- arrow-ipc: Write 0 offset buffer for length-0 variable-size arrays [\#9717](https://github.com/apache/arrow-rs/pull/9717) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([atwam](https://github.com/atwam)) -- \[Json\] Support `FixedSizeList` in json decoder [\#9715](https://github.com/apache/arrow-rs/pull/9715) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) -- chore\(deps\): bump actions/upload-pages-artifact from 4 to 5 [\#9713](https://github.com/apache/arrow-rs/pull/9713) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Fix clippy warning in fixed\_size\_binary\_array.rs [\#9712](https://github.com/apache/arrow-rs/pull/9712) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([AdamGS](https://github.com/AdamGS)) -- feat: add `has_non_empty_nulls` helper function in `OffsetBuffer` [\#9711](https://github.com/apache/arrow-rs/pull/9711) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) -- chore\(deps\): bump pytest from 7.2.0 to 9.0.3 in /parquet/pytest [\#9706](https://github.com/apache/arrow-rs/pull/9706) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([dependabot[bot]](https://github.com/apps/dependabot)) -- Fedora license audit [\#9704](https://github.com/apache/arrow-rs/pull/9704) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([michel-slm](https://github.com/michel-slm)) -- \[Variant\] Take top-level nulls into consideration when extracting perfectly shredded children [\#9702](https://github.com/apache/arrow-rs/pull/9702) ([AdamGS](https://github.com/AdamGS)) -- feat\(parquet\): add `push_decoder` benchmark for `PushBuffers` overhead [\#9696](https://github.com/apache/arrow-rs/pull/9696) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) -- Add mutable bitwise operations to `BooleanArray` and `NullBuffer::union_many` [\#9692](https://github.com/apache/arrow-rs/pull/9692) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([mbutrovich](https://github.com/mbutrovich)) -- chore\(deps\): update hashbrown requirement from 0.16.0 to 0.17.0 [\#9691](https://github.com/apache/arrow-rs/pull/9691) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([dependabot[bot]](https://github.com/apps/dependabot)) -- chore\(deps\): bump actions/github-script from 8 to 9 [\#9690](https://github.com/apache/arrow-rs/pull/9690) ([dependabot[bot]](https://github.com/apps/dependabot)) -- minor: Re-enable CDC bench [\#9686](https://github.com/apache/arrow-rs/pull/9686) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://github.com/etseidl)) -- \[Variant\] Add `VariantArrayBuilder::append_nulls` API [\#9685](https://github.com/apache/arrow-rs/pull/9685) ([sdf-jkl](https://github.com/sdf-jkl)) -- feat\(parquet\): add struct-column writer benchmarks [\#9679](https://github.com/apache/arrow-rs/pull/9679) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) -- \[Arrow\] Add API to check if `Field` has a valid `ExtensionType` [\#9677](https://github.com/apache/arrow-rs/pull/9677) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([sdf-jkl](https://github.com/sdf-jkl)) -- \[Variant\] `variant_get` should follow JSONPath semantics for Field path element [\#9676](https://github.com/apache/arrow-rs/pull/9676) ([sdf-jkl](https://github.com/sdf-jkl)) -- ParquetMetaDataPushDecoder API to clear all buffered ranges [\#9673](https://github.com/apache/arrow-rs/pull/9673) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([nathanb9](https://github.com/nathanb9)) -- Fix union cast incorrectness for duplicate field names [\#9666](https://github.com/apache/arrow-rs/pull/9666) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([friendlymatthew](https://github.com/friendlymatthew)) -- chore: re-export `MAX_INLINE_VIEW_LEN` from `arrow_data` [\#9665](https://github.com/apache/arrow-rs/pull/9665) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rluvaton](https://github.com/rluvaton)) -- No longer allow BIT\_PACKED level encoding in Parquet writer [\#9656](https://github.com/apache/arrow-rs/pull/9656) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://github.com/etseidl)) -- feat\(parquet\): add sparse-column writer benchmarks [\#9654](https://github.com/apache/arrow-rs/pull/9654) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) -- Support `GenericListViewArray::new_unchecked` and refactor `ListView` json decoder [\#9648](https://github.com/apache/arrow-rs/pull/9648) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) -- \[Json\] Add json reader benchmarks for ListView [\#9647](https://github.com/apache/arrow-rs/pull/9647) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) -- fix\(parquet\): fix CDC panic on nested ListArrays with null entries [\#9644](https://github.com/apache/arrow-rs/pull/9644) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([kszucs](https://github.com/kszucs)) -- Add a test for reading nested REE data in json [\#9634](https://github.com/apache/arrow-rs/pull/9634) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) -- \[Variant\] Fix `variant_get` to return `List` instead of `List` [\#9631](https://github.com/apache/arrow-rs/pull/9631) ([liamzwbao](https://github.com/liamzwbao)) -- ci: use ubuntu-slim runner for lightweight CI jobs [\#9630](https://github.com/apache/arrow-rs/pull/9630) ([CuteChuanChuan](https://github.com/CuteChuanChuan)) -- Add bloom filter folding to automatically size SBBF filters [\#9628](https://github.com/apache/arrow-rs/pull/9628) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([adriangb](https://github.com/adriangb)) -- Add List and ListView take benchmarks [\#9626](https://github.com/apache/arrow-rs/pull/9626) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([AdamGS](https://github.com/AdamGS)) -- ParquetPushDecoder API to clear all buffered ranges [\#9624](https://github.com/apache/arrow-rs/pull/9624) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([nathanb9](https://github.com/nathanb9)) -- fix: handle missing dictionary batch for null-only columns in IPC reader [\#9623](https://github.com/apache/arrow-rs/pull/9623) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([joaquinhuigomez](https://github.com/joaquinhuigomez)) -- Fix `MutableBuffer::clear` [\#9622](https://github.com/apache/arrow-rs/pull/9622) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Rafferty97](https://github.com/Rafferty97)) -- feat\[arrow-ord\]: suppport REE comparisons [\#9621](https://github.com/apache/arrow-rs/pull/9621) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([asubiotto](https://github.com/asubiotto)) -- chore\(deps\): update sha2 requirement from 0.10 to 0.11 [\#9618](https://github.com/apache/arrow-rs/pull/9618) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([dependabot[bot]](https://github.com/apps/dependabot)) -- Expose option to set line terminator for CSV writer [\#9617](https://github.com/apache/arrow-rs/pull/9617) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([svranesevic](https://github.com/svranesevic)) -- \[Json\] Add json reader benchmarks for Map and REE [\#9616](https://github.com/apache/arrow-rs/pull/9616) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) -- deps: fix `object_store` breakage for 0.13.2 [\#9612](https://github.com/apache/arrow-rs/pull/9612) ([mzabaluev-flarion](https://github.com/mzabaluev-flarion)) -- \[Variant\] Support Binary/LargeBinary children [\#9610](https://github.com/apache/arrow-rs/pull/9610) ([AdamGS](https://github.com/AdamGS)) -- fix: use writer types in Skipper for resolved named record types [\#9605](https://github.com/apache/arrow-rs/pull/9605) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([ariel-miculas](https://github.com/ariel-miculas)) -- feat\(parquet\): derive `PartialEq` and `Eq` for `CdcOptions` [\#9602](https://github.com/apache/arrow-rs/pull/9602) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([kszucs](https://github.com/kszucs)) -- Add `finish_preserve_values` to `ArrayBuilder` trait [\#9601](https://github.com/apache/arrow-rs/pull/9601) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([adamreichold](https://github.com/adamreichold)) -- \[Variant\] extend shredded null handling for arrays [\#9599](https://github.com/apache/arrow-rs/pull/9599) ([sdf-jkl](https://github.com/sdf-jkl)) -- \[Variant\] Add unshredded `Struct` fast-path for `variant_get(..., Struct)` [\#9597](https://github.com/apache/arrow-rs/pull/9597) ([sdf-jkl](https://github.com/sdf-jkl)) -- Pre-reserve output capacity in ByteView/ByteArray dictionary decoding [\#9590](https://github.com/apache/arrow-rs/pull/9590) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([Dandandan](https://github.com/Dandandan)) -- \[Variant\] Align cast logic for variant\_get to cast kernel for numeric/bool types [\#9563](https://github.com/apache/arrow-rs/pull/9563) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([klion26](https://github.com/klion26)) -- Add support to cast from `UnionArray` [\#9544](https://github.com/apache/arrow-rs/pull/9544) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([friendlymatthew](https://github.com/friendlymatthew)) -- Support `ListView` codec in arrow-json [\#9503](https://github.com/apache/arrow-rs/pull/9503) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- impl `FromStr` for `DatePart` [\#9931](https://github.com/apache/arrow-rs/pull/9931) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([sdf-jkl](https://github.com/sdf-jkl)) +- Pre-size dedup HashTable in GenericByteDictionaryBuilder::with\_capacity [\#9908](https://github.com/apache/arrow-rs/pull/9908) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([rabenhorst](https://github.com/rabenhorst)) +- \[arrow-array\] Use consistent `value_length` name in FixedSizeBinaryArray [\#9905](https://github.com/apache/arrow-rs/pull/9905) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- replace Dictionary::try\_new\(\) calls with with\_values. [\#9894](https://github.com/apache/arrow-rs/pull/9894) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Rich-T-kid](https://github.com/Rich-T-kid)) +- API to help with the pattern of 'replaces the values of the REE array [\#9891](https://github.com/apache/arrow-rs/pull/9891) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Rich-T-kid](https://github.com/Rich-T-kid)) +- fix\(arrow-csv\): bound RecordDecoder::flush offset accumulation [\#9886](https://github.com/apache/arrow-rs/pull/9886) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([masumi-ryugo](https://github.com/masumi-ryugo)) +- fix\(parquet\): bound schema num\_children before Vec::with\_capacity [\#9884](https://github.com/apache/arrow-rs/pull/9884) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([masumi-ryugo](https://github.com/masumi-ryugo)) +- feat\(arrow-string\): concat\_elements for view, fixed binary [\#9876](https://github.com/apache/arrow-rs/pull/9876) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([theirix](https://github.com/theirix)) +- Prevent `FixedSizeBinaryArray` `i32` offset overflows \(try 2\) [\#9872](https://github.com/apache/arrow-rs/pull/9872) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- \[arrow-string\]: add `like::eq_ascii_ignore_case` kernel [\#9871](https://github.com/apache/arrow-rs/pull/9871) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([albertlockett](https://github.com/albertlockett)) +- fix\(parquet\): Prevent negative list sizes in Thrift compact protocol parser [\#9868](https://github.com/apache/arrow-rs/pull/9868) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([masumi-ryugo](https://github.com/masumi-ryugo)) +- \[PARQUET\] Allow `UNKNOWN` logical type annotation on any physical type [\#9855](https://github.com/apache/arrow-rs/pull/9855) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://github.com/etseidl)) +- \[arrow-ipc\]: dictionary builders for delta - doc fix and integration tests for nested types [\#9853](https://github.com/apache/arrow-rs/pull/9853) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([albertlockett](https://github.com/albertlockett)) +- fix\(arrow-cast\): fix incorrect conversion [\#9852](https://github.com/apache/arrow-rs/pull/9852) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([bboissin](https://github.com/bboissin)) +- chore\[benches\]: add REE interleave benchmarks [\#9849](https://github.com/apache/arrow-rs/pull/9849) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([asubiotto](https://github.com/asubiotto)) +- test\(parquet\): replace `InMemoryArrayReader` with `PrimitiveArrayReader` in tests [\#9847](https://github.com/apache/arrow-rs/pull/9847) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([HippoBaro](https://github.com/HippoBaro)) +- REE row conversion speed up [\#9845](https://github.com/apache/arrow-rs/pull/9845) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([Rich-T-kid](https://github.com/Rich-T-kid)) +- fix\(parquet\): Avoid panic on malformed thrift bool fields in parquet metadata [\#9840](https://github.com/apache/arrow-rs/pull/9840) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([BoazC-MSFT](https://github.com/BoazC-MSFT)) +- fix\(parquet\): avoid panic on ColumnIndex length mismatch [\#9833](https://github.com/apache/arrow-rs/pull/9833) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([pchintar](https://github.com/pchintar)) +- configurable data page v2 compression threshold [\#9826](https://github.com/apache/arrow-rs/pull/9826) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([leoyvens](https://github.com/leoyvens)) +- Prevent `ArrayData::slice` length overflow [\#9813](https://github.com/apache/arrow-rs/pull/9813) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([alamb](https://github.com/alamb)) +- Fix JSON reader panic for non-nullable zero-size FixedSizeList [\#9810](https://github.com/apache/arrow-rs/pull/9810) [[arrow](https://github.com/apache/arrow-rs/labels/arrow)] ([liamzwbao](https://github.com/liamzwbao)) +- \[Parquet\] Do not panic when trying to skip records in delta encoded files using non-standard block sizes [\#9794](https://github.com/apache/arrow-rs/pull/9794) [[parquet](https://github.com/apache/arrow-rs/labels/parquet)] ([etseidl](https://github.com/etseidl)) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 83ee468c32e5..48075c5ef771 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -256,3 +256,21 @@ If sometimes you want to commit without checking, just run `git commit` with `-- ```bash git commit --no-verify -m "... commit message ..." ``` + +## AI Generated Submissions + +This project follows the guidance for AI generated submissions used by the +[Arrow Project](https://arrow.apache.org/docs/dev/developers/overview.html#ai-generated-code). +As such, it is expected that you will: + +- Only submit a PR if you are able to debug and own the changes yourself - review all generated + code to understand every detail +- Match the style and conventions used in the rest of the codebase, including PR titles and descriptions +- Be upfront about AI usage and summarise what was AI-generated +- If there are parts you don’t fully understand, leave comments on your own PR explaining what steps you took to verify correctness +- Watch for AI’s tendency to generate overly verbose comments, unnecessary test cases, and incorrect fixes +- Break down large PRs into smaller ones to make review easier + +It is also important for submitters to be aware of potential copyright issues. See the ASF's +[guidance](https://www.apache.org/legal/generative-tooling.html) on AI-generated code for further +information on licensing considerations. diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 000000000000..11979eb5836b --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,4598 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.60.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.60.2", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "apache-avro" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" +dependencies = [ + "bigdecimal", + "bon", + "digest 0.10.7", + "log", + "miniz_oxide", + "num-bigint", + "quad-rand", + "rand 0.9.4", + "regex-lite", + "serde", + "serde_bytes", + "serde_json", + "strum", + "strum_macros 0.27.2", + "thiserror 2.0.18", + "uuid", +] + +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "arrow" +version = "58.3.0" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-pyarrow", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", + "bytes", + "chrono", + "criterion", + "half", + "memmap2", + "rand 0.9.4", + "serde", +] + +[[package]] +name = "arrow-arith" +version = "58.3.0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "58.3.0" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "criterion", + "futures", + "half", + "hashbrown 0.17.0", + "num-complex", + "num-integer", + "num-traits", + "rand 0.9.4", +] + +[[package]] +name = "arrow-avro" +version = "58.3.0" +dependencies = [ + "apache-avro", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "async-stream", + "bytes", + "bzip2", + "crc", + "criterion", + "flate2", + "futures", + "half", + "indexmap", + "liblzma", + "md5", + "num-bigint", + "object_store", + "once_cell", + "rand 0.9.4", + "serde", + "serde_json", + "sha2", + "snap", + "strum_macros 0.28.0", + "tempfile", + "tokio", + "uuid", + "zstd", +] + +[[package]] +name = "arrow-buffer" +version = "58.3.0" +dependencies = [ + "bytes", + "criterion", + "half", + "num-bigint", + "num-traits", + "rand 0.9.4", +] + +[[package]] +name = "arrow-cast" +version = "58.3.0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "comfy-table", + "criterion", + "half", + "insta", + "lexical-core", + "num-traits", + "rand 0.9.4", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "58.3.0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-schema", + "bytes", + "chrono", + "csv", + "csv-core", + "futures", + "regex", + "tempfile", + "tokio", +] + +[[package]] +name = "arrow-data" +version = "58.3.0" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-flight" +version = "58.3.0" +dependencies = [ + "anyhow", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", + "assert_cmd", + "base64", + "bytes", + "clap", + "futures", + "http", + "http-body", + "hyper-util", + "once_cell", + "paste", + "pin-project-lite", + "prost", + "prost-types", + "tempfile", + "tokio", + "tokio-stream", + "tonic", + "tonic-prost", + "tower", + "tracing-log", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "arrow-integration-test" +version = "58.3.0" +dependencies = [ + "arrow", + "arrow-buffer", + "hex", + "num-bigint", + "num-traits", + "serde", + "serde_json", +] + +[[package]] +name = "arrow-integration-testing" +version = "58.3.0" +dependencies = [ + "arrow", + "arrow-buffer", + "arrow-flight", + "arrow-integration-test", + "clap", + "flate2", + "futures", + "prost", + "serde_json", + "tempfile", + "tokio", + "tonic", + "tracing-subscriber", +] + +[[package]] +name = "arrow-ipc" +version = "58.3.0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "bytes", + "criterion", + "flatbuffers", + "lz4_flex", + "memmap2", + "tempfile", + "tokio", + "zstd", +] + +[[package]] +name = "arrow-json" +version = "58.3.0" +dependencies = [ + "arbitrary", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "bytes", + "chrono", + "criterion", + "flate2", + "futures", + "half", + "indexmap", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "rand 0.9.4", + "ryu", + "serde", + "serde_core", + "serde_json", + "simdutf8", + "tokio", +] + +[[package]] +name = "arrow-ord" +version = "58.3.0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "half", + "rand 0.9.4", +] + +[[package]] +name = "arrow-pyarrow" +version = "58.3.0" +dependencies = [ + "arrow-array", + "arrow-data", + "arrow-schema", + "pyo3", +] + +[[package]] +name = "arrow-row" +version = "58.3.0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "half", + "rand 0.9.4", +] + +[[package]] +name = "arrow-schema" +version = "58.3.0" +dependencies = [ + "bitflags", + "criterion", + "insta", + "postcard", + "serde", + "serde_core", + "serde_json", +] + +[[package]] +name = "arrow-select" +version = "58.3.0" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", + "rand 0.9.4", +] + +[[package]] +name = "arrow-string" +version = "58.3.0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "assert_cmd" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39bae1d3fa576f7c6519514180a72559268dd7d1fe104070956cb687bc6673bd" +dependencies = [ + "anstyle", + "bstr", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "aws-lc-rs" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bigdecimal" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "bitflags" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bon" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + +[[package]] +name = "brotli" +version = "8.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.2.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d16d90359e986641506914ba71350897565610e87ce0ad9e6f28569db3dd5c6d" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "num-traits", + "serde", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", + "terminal_size", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "console" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" +dependencies = [ + "encode_unicode", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "futures", + "itertools 0.13.0", + "num-traits", + "oorandom", + "page_size", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools 0.13.0", +] + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.0", + "const-oid", + "crypto-common 0.2.1", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "embedded-io" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "gen" +version = "0.1.0" +dependencies = [ + "prost-build", + "tonic-prost-build", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "geo-traits" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7c353d12a704ccfab1ba8bfb1a7fe6cb18b665bf89d37f4f7890edcd260206" +dependencies = [ + "geo-types", +] + +[[package]] +name = "geo-types" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94776032c45f950d30a13af6113c2ad5625316c9abfbccee4dd5a6695f8fe0f5" +dependencies = [ + "approx", + "num-traits", + "serde", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasip2", + "wasip3", +] + +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "rand 0.9.4", + "rand_distr", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + +[[package]] +name = "hybrid-array" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d46837a0ed51fe95bd3b05de33cd64a1ee88fc797477ca48446872504507c5" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.0", + "serde", + "serde_core", +] + +[[package]] +name = "insta" +version = "1.47.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4a6248eb93a4401ed2f37dfe8ea592d3cf05b7cf4f8efa867b6895af7e094e" +dependencies = [ + "console", + "once_cell", + "similar", + "tempfile", +] + +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" +dependencies = [ + "cfg-if", + "futures-util", + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libbz2-rs-sys" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3a6a8c165077efc8f3a971534c50ea6a1a18b329ef4a66e897a7e3a1494565f" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "liblzma" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6033b77c21d1f56deeae8014eb9fbe7bdf1765185a6c508b5ca82eeaed7f899" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a60851d15cd8c5346eca4ab8babff585be2ae4bc8097c067291d3ffe2add3b6" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "lz4_flex" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md5" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.60.2", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "object_store" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +dependencies = [ + "async-trait", + "base64", + "bytes", + "chrono", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body-util", + "httparse", + "humantime", + "hyper", + "itertools 0.14.0", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.10.1", + "reqwest", + "ring", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "parquet" +version = "58.3.0" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64", + "brotli", + "bytes", + "chrono", + "clap", + "crc32fast", + "criterion", + "flate2", + "futures", + "half", + "hashbrown 0.17.0", + "insta", + "lz4_flex", + "num-bigint", + "num-integer", + "num-traits", + "object_store", + "parquet-geospatial", + "parquet-variant", + "parquet-variant-compute", + "parquet-variant-json", + "paste", + "rand 0.9.4", + "ring", + "seq-macro", + "serde", + "serde_json", + "simdutf8", + "snap", + "sysinfo", + "tempfile", + "thrift", + "tokio", + "twox-hash", + "zstd", +] + +[[package]] +name = "parquet-geospatial" +version = "58.3.0" +dependencies = [ + "arrow-schema", + "geo-traits", + "serde", + "serde_json", + "wkb", + "wkt", +] + +[[package]] +name = "parquet-variant" +version = "58.3.0" +dependencies = [ + "arrow", + "arrow-schema", + "chrono", + "criterion", + "half", + "indexmap", + "num-traits", + "paste", + "rand 0.9.4", + "simdutf8", + "uuid", +] + +[[package]] +name = "parquet-variant-compute" +version = "58.3.0" +dependencies = [ + "arrow", + "arrow-schema", + "chrono", + "criterion", + "half", + "indexmap", + "parquet-variant", + "parquet-variant-json", + "rand 0.9.4", + "serde_json", + "uuid", +] + +[[package]] +name = "parquet-variant-json" +version = "58.3.0" +dependencies = [ + "arrow-schema", + "base64", + "chrono", + "parquet-variant", + "serde_json", + "uuid", +] + +[[package]] +name = "parquet_derive" +version = "58.3.0" +dependencies = [ + "parquet", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "parquet_derive_test" +version = "58.3.0" +dependencies = [ + "chrono", + "parquet", + "parquet_derive", + "uuid", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf0d9e68100b3a7989b4901972f265cd542e560a3a8a724e1e20322f4d06ce9" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a990e22f43e84855daf260dded30524ef4a9021cc7541c26540500a50b624389" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "embedded-io 0.4.0", + "embedded-io 0.6.1", + "serde", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "difflib", + "predicates-core", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + +[[package]] +name = "pyo3" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", +] + +[[package]] +name = "pyo3-build-config" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.28.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "quad-rand" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" + +[[package]] +name = "quick-xml" +version = "0.39.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "721da970c312655cde9b4ffe0547f20a8494866a4af5ff51f18b7c633d0c870b" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.4", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.4", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "aws-lc-rs", + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sysinfo" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.52.0", +] + +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix", + "windows-sys 0.60.2", +] + +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "110a78583f19d5cdb2c5ccf321d1290344e71313c6c37d43520d386027d18386" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.11+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "tonic" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" +dependencies = [ + "async-trait", + "axum", + "base64", + "bytes", + "flate2", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "rustls-native-certs", + "socket2", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", + "webpki-roots", + "zstd", +] + +[[package]] +name = "tonic-build" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1882ac3bf5ef12877d7ed57aad87e75154c11931c2ba7e6cde5e22d63522c734" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tonic-prost" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3144df636917574672e93d0f56d7edec49f90305749c668df5101751bb8f95a" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn", + "tempfile", + "tonic-build", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "thread_local", + "tracing", + "tracing-core", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + +[[package]] +name = "typenum" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.121" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winnow" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "wkb" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a120b336c7ad17749026d50427c23d838ecb50cd64aaea6254b5030152f890a9" +dependencies = [ + "byteorder", + "geo-traits", + "num_enum", + "thiserror 1.0.69", +] + +[[package]] +name = "wkt" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efb2b923ccc882312e559ffaa832a055ba9d1ac0cc8e86b3e25453247e4b81d7" +dependencies = [ + "geo-traits", + "geo-types", + "log", + "num-traits", + "thiserror 1.0.69", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zlib-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 52140a05287c..9a13149626e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,7 +68,7 @@ exclude = [ ] [workspace.package] -version = "58.2.0" +version = "58.3.0" homepage = "https://github.com/apache/arrow-rs" repository = "https://github.com/apache/arrow-rs" authors = ["Apache Arrow "] @@ -85,26 +85,26 @@ edition = "2024" rust-version = "1.85" [workspace.dependencies] -arrow = { version = "58.2.0", path = "./arrow", default-features = false } -arrow-arith = { version = "58.2.0", path = "./arrow-arith" } -arrow-array = { version = "58.2.0", path = "./arrow-array" } -arrow-buffer = { version = "58.2.0", path = "./arrow-buffer" } -arrow-cast = { version = "58.2.0", path = "./arrow-cast" } -arrow-csv = { version = "58.2.0", path = "./arrow-csv" } -arrow-data = { version = "58.2.0", path = "./arrow-data" } -arrow-ipc = { version = "58.2.0", path = "./arrow-ipc" } -arrow-json = { version = "58.2.0", path = "./arrow-json" } -arrow-ord = { version = "58.2.0", path = "./arrow-ord" } -arrow-pyarrow = { version = "58.2.0", path = "./arrow-pyarrow" } -arrow-row = { version = "58.2.0", path = "./arrow-row" } -arrow-schema = { version = "58.2.0", path = "./arrow-schema" } -arrow-select = { version = "58.2.0", path = "./arrow-select" } -arrow-string = { version = "58.2.0", path = "./arrow-string" } -parquet = { version = "58.2.0", path = "./parquet", default-features = false } -parquet-geospatial = { version = "58.2.0", path = "./parquet-geospatial" } -parquet-variant = { version = "58.2.0", path = "./parquet-variant" } -parquet-variant-json = { version = "58.2.0", path = "./parquet-variant-json" } -parquet-variant-compute = { version = "58.2.0", path = "./parquet-variant-compute" } +arrow = { version = "58.3.0", path = "./arrow", default-features = false } +arrow-arith = { version = "58.3.0", path = "./arrow-arith" } +arrow-array = { version = "58.3.0", path = "./arrow-array" } +arrow-buffer = { version = "58.3.0", path = "./arrow-buffer" } +arrow-cast = { version = "58.3.0", path = "./arrow-cast" } +arrow-csv = { version = "58.3.0", path = "./arrow-csv" } +arrow-data = { version = "58.3.0", path = "./arrow-data" } +arrow-ipc = { version = "58.3.0", path = "./arrow-ipc" } +arrow-json = { version = "58.3.0", path = "./arrow-json" } +arrow-ord = { version = "58.3.0", path = "./arrow-ord" } +arrow-pyarrow = { version = "58.3.0", path = "./arrow-pyarrow" } +arrow-row = { version = "58.3.0", path = "./arrow-row" } +arrow-schema = { version = "58.3.0", path = "./arrow-schema" } +arrow-select = { version = "58.3.0", path = "./arrow-select" } +arrow-string = { version = "58.3.0", path = "./arrow-string" } +parquet = { version = "58.3.0", path = "./parquet", default-features = false } +parquet-geospatial = { version = "58.3.0", path = "./parquet-geospatial" } +parquet-variant = { version = "58.3.0", path = "./parquet-variant" } +parquet-variant-json = { version = "58.3.0", path = "./parquet-variant-json" } +parquet-variant-compute = { version = "58.3.0", path = "./parquet-variant-compute" } chrono = { version = "0.4.40", default-features = false, features = ["clock"] } diff --git a/README.md b/README.md index 0735dae57cdc..7ea4205d56f5 100644 --- a/README.md +++ b/README.md @@ -91,13 +91,23 @@ Planned Release Schedule | Approximate Date | Version | Notes | | ---------------- | ---------- | --------------------------------------- | -| March 2026 | [`58.1.0`] | Minor, NO breaking API changes | -| April 2026 | [`58.2.0`] | Minor, NO breaking API changes | +| May 2026 | [`58.3.0`] | Minor, NO breaking API changes | +| May 2026 | [`57.3.1`] | Patch, NO breaking API changes | +| May 2026 | [`56.2.1`] | Patch, NO breaking API changes | | May 2026 | [`59.0.0`] | Major, potentially breaking API changes | +| June 2026 | [`59.1.0`] | Minor, NO breaking API changes | +| July 2026 | [`59.2.0`] | Minor, NO breaking API changes | +| August 2026 | [`60.0.0`] | Major, potentially breaking API changes | [`58.1.0`]: https://github.com/apache/arrow-rs/issues/9108 [`58.2.0`]: https://github.com/apache/arrow-rs/issues/9109 +[`58.3.0`]: https://github.com/apache/arrow-rs/issues/9859 +[`57.3.1`]: https://github.com/apache/arrow-rs/issues/9858 +[`56.2.1`]: https://github.com/apache/arrow-rs/issues/9857 [`59.0.0`]: https://github.com/apache/arrow-rs/issues/9110 +[`59.1.0`]: https://github.com/apache/arrow-rs/issues/9878 +[`59.2.0`]: https://github.com/apache/arrow-rs/issues/9879 +[`60.0.0`]: https://github.com/apache/arrow-rs/issues/9880 [ticket #5368]: https://github.com/apache/arrow-rs/issues/5368 [semantic versioning]: https://semver.org/ diff --git a/arrow-arith/src/temporal.rs b/arrow-arith/src/temporal.rs index 12970f4498fa..96a3b43a3811 100644 --- a/arrow-arith/src/temporal.rs +++ b/arrow-arith/src/temporal.rs @@ -17,12 +17,14 @@ //! Defines temporal kernels for time and date related functions. +use std::str::FromStr; use std::sync::Arc; use arrow_array::cast::AsArray; use cast::as_primitive_array; use chrono::{Datelike, TimeZone, Timelike, Utc}; +use arrow_array::ree_map; use arrow_array::temporal_conversions::{ MICROSECONDS, MICROSECONDS_IN_DAY, MILLISECONDS, MILLISECONDS_IN_DAY, NANOSECONDS, NANOSECONDS_IN_DAY, SECONDS_IN_DAY, date32_to_datetime, date64_to_datetime, @@ -83,6 +85,59 @@ impl std::fmt::Display for DatePart { } } +/// Parses a string into a [`DatePart`]. +/// +/// Matching is case-insensitive. The accepted names follow PostgreSQL's +/// `EXTRACT` / `date_part` conventions (see +/// ) +/// and are aligned with the alias set accepted by `arrow-cast`'s +/// `IntervalUnit::from_str` (plurals, short forms like `y`, `mon`, `ms`, +/// `us`) so the two parsers round-trip on the names they share. +/// +/// Names that do not map one-to-one onto a [`DatePart`] variant are +/// intentionally not recognized: +/// - `epoch` — a derived computation, not a date part. +/// - `century`, `decade`, `millennium` — no matching [`DatePart`] variant. +/// - `timezone`, `timezone_hour`, `timezone_minute` — not modeled by +/// [`DatePart`]. +/// - `isodow` — PostgreSQL's `isodow` returns `1..=7` (Mon=1), but the +/// closest variant ([`DatePart::DayOfWeekMonday0`]) returns `0..=6`. +/// Accepting it here would silently shift the value range; callers that +/// need the PostgreSQL semantic should map the alias themselves and +/// add `1` to the extracted value. +impl FromStr for DatePart { + type Err = ArrowError; + + fn from_str(s: &str) -> Result { + Ok(match s.to_lowercase().as_str() { + "y" | "yr" | "yrs" | "year" | "years" => Self::Year, + "isoyear" => Self::YearISO, + "qtr" | "quarter" | "quarters" => Self::Quarter, + "mon" | "mons" | "month" | "months" => Self::Month, + "w" | "week" | "weeks" => Self::Week, + "isoweek" => Self::WeekISO, + "d" | "day" | "days" => Self::Day, + "dow" | "dayofweek" => Self::DayOfWeekSunday0, + "doy" | "dayofyear" => Self::DayOfYear, + "h" | "hr" | "hrs" | "hour" | "hours" => Self::Hour, + "m" | "min" | "mins" | "minute" | "minutes" => Self::Minute, + "s" | "sec" | "secs" | "second" | "seconds" => Self::Second, + "ms" | "msec" | "msecs" | "msecond" | "mseconds" | "millisecond" | "milliseconds" => { + Self::Millisecond + } + "us" | "usec" | "usecs" | "usecond" | "useconds" | "microsecond" | "microseconds" => { + Self::Microsecond + } + "nanosecond" | "nanoseconds" => Self::Nanosecond, + _ => { + return Err(ArrowError::InvalidArgumentError(format!( + "Unknown date part: {s}" + ))); + } + }) + } +} + /// Returns function to extract relevant [`DatePart`] from types like a /// [`NaiveDateTime`] or [`DateTime`]. /// @@ -194,6 +249,15 @@ pub fn date_part(array: &dyn Array, part: DatePart) -> Result match k.data_type() { + DataType::Int16 => ree_map!(array, Int16Type, |a| date_part(a, part)), + DataType::Int32 => ree_map!(array, Int32Type, |a| date_part(a, part)), + DataType::Int64 => ree_map!(array, Int64Type, |a| date_part(a, part)), + _ => Err(ArrowError::InvalidArgumentError(format!( + "Invalid run-end type: {:?}", + k.data_type() + ))), + }, t => return_compute_error_with!(format!("{part} does not support"), t), ) } @@ -2040,4 +2104,121 @@ mod tests { assert_eq!(2015, actual.value(1)); assert_eq!(2016, actual.value(2)); } + + #[test] + fn test_ree_timestamp_year() { + let vals: TimestampSecondArray = + vec![Some(1514764800), Some(1550636625), Some(1550636625)].into(); + let run_ends = Int32Array::from(vec![1, 2, 3]); + let ree = RunArray::try_new(&run_ends, &vals).unwrap(); + + let b = date_part(&ree, DatePart::Year).unwrap(); + let ree_result = b.as_run_opt::().unwrap(); + let values = ree_result.values().as_primitive::(); + assert_eq!(2018, values.value(0)); + assert_eq!(2019, values.value(1)); + assert_eq!(2019, values.value(2)); + } + + #[test] + fn test_ree_date64_month() { + let vals: PrimitiveArray = + vec![Some(1514764800000), Some(1550636625000)].into(); + let run_ends = Int64Array::from(vec![2, 4]); + let ree = RunArray::try_new(&run_ends, &vals).unwrap(); + + let b = date_part(&ree, DatePart::Month).unwrap(); + let ree_result = b.as_run_opt::().unwrap(); + let values = ree_result.values().as_primitive::(); + assert_eq!(1, values.value(0)); + assert_eq!(2, values.value(1)); + } + + #[test] + fn test_date_part_from_str() { + let cases = [ + ("y", DatePart::Year), + ("yr", DatePart::Year), + ("year", DatePart::Year), + ("years", DatePart::Year), + ("YEAR", DatePart::Year), + ("isoyear", DatePart::YearISO), + ("qtr", DatePart::Quarter), + ("quarter", DatePart::Quarter), + ("quarters", DatePart::Quarter), + ("mon", DatePart::Month), + ("mons", DatePart::Month), + ("month", DatePart::Month), + ("MONTHS", DatePart::Month), + ("w", DatePart::Week), + ("week", DatePart::Week), + ("isoweek", DatePart::WeekISO), + ("d", DatePart::Day), + ("day", DatePart::Day), + ("dow", DatePart::DayOfWeekSunday0), + ("DayOfWeek", DatePart::DayOfWeekSunday0), + ("doy", DatePart::DayOfYear), + ("DayOfYear", DatePart::DayOfYear), + ("h", DatePart::Hour), + ("hr", DatePart::Hour), + ("hour", DatePart::Hour), + // Pin the famous `m`/`mon` ambiguity: `m` resolves to Minute, + // matching `arrow-cast`'s `IntervalUnit::from_str`. Use `mon` + // (or `month`) for Month. + ("m", DatePart::Minute), + ("min", DatePart::Minute), + ("mins", DatePart::Minute), + ("minute", DatePart::Minute), + ("s", DatePart::Second), + ("sec", DatePart::Second), + ("second", DatePart::Second), + ("ms", DatePart::Millisecond), + ("msec", DatePart::Millisecond), + ("mseconds", DatePart::Millisecond), + ("millisecond", DatePart::Millisecond), + ("us", DatePart::Microsecond), + ("usec", DatePart::Microsecond), + ("useconds", DatePart::Microsecond), + ("microsecond", DatePart::Microsecond), + ("nanosecond", DatePart::Nanosecond), + ("nanoseconds", DatePart::Nanosecond), + ]; + for (input, expected) in cases { + assert_eq!( + DatePart::from_str(input).unwrap(), + expected, + "parsing {input}" + ); + } + } + + #[test] + fn test_date_part_from_str_unknown() { + // Names intentionally rejected — see the FromStr doc comment for why. + // `isodow` is here because mapping it to `DayOfWeekMonday0` would + // silently shift the value range vs. PostgreSQL's `isodow` (1..=7). + let unknown = [ + "epoch", + "century", + "decade", + "millennium", + "timezone", + "timezone_hour", + "timezone_minute", + "isodow", + // Whitespace is not trimmed — pin this so the behavior doesn't + // change silently. Callers should trim before parsing. + " year ", + "year ", + "", + "nope", + ]; + for s in unknown { + let err = DatePart::from_str(s).unwrap_err(); + assert!( + matches!(err, ArrowError::InvalidArgumentError(_)), + "expected InvalidArgumentError for {s:?}, got: {err}" + ); + } + } } diff --git a/arrow-array/src/array/fixed_size_binary_array.rs b/arrow-array/src/array/fixed_size_binary_array.rs index 928aa38a2727..a54496addc5d 100644 --- a/arrow-array/src/array/fixed_size_binary_array.rs +++ b/arrow-array/src/array/fixed_size_binary_array.rs @@ -96,42 +96,48 @@ pub struct FixedSizeBinaryArray { } impl FixedSizeBinaryArray { - /// Create a new [`FixedSizeBinaryArray`] with `size` element size, panicking on failure + /// Create a new [`FixedSizeBinaryArray`] with `value_length` bytes per element, panicking on + /// failure /// /// # Panics /// /// Panics if [`Self::try_new`] returns an error - pub fn new(size: i32, values: Buffer, nulls: Option) -> Self { - Self::try_new(size, values, nulls).unwrap() + pub fn new(value_length: i32, values: Buffer, nulls: Option) -> Self { + Self::try_new(value_length, values, nulls).unwrap() } /// Create a new [`Scalar`] from `value` pub fn new_scalar(value: impl AsRef<[u8]>) -> Scalar { let v = value.as_ref(); - Scalar::new(Self::new(v.len() as _, Buffer::from(v), None)) + let value_length = + i32::try_from(v.len()).expect("FixedSizeBinaryArray value length exceeds i32"); + Scalar::new(Self::new(value_length, Buffer::from(v), None)) } /// Create a new [`FixedSizeBinaryArray`] from the provided parts, returning an error on failure /// - /// Creating an arrow with `size == 0` will try to get the length from the null buffer. If - /// no null buffer is provided, the resulting array will have length zero. + /// Creating an array with `value_length == 0` will try to get the length from the null + /// buffer. If no null buffer is provided, the resulting array will have length zero. /// /// # Errors /// - /// * `size < 0` - /// * `values.len() / size != nulls.len()` - /// * `size == 0 && values.len() != 0` + /// * `value_length < 0` + /// * `values.len() / value_length != nulls.len()` + /// * `value_length == 0 && values.len() != 0` + /// * `len * value_length > i32::MAX` pub fn try_new( - size: i32, + value_length: i32, values: Buffer, nulls: Option, ) -> Result { - let data_type = DataType::FixedSizeBinary(size); - let s = size.to_usize().ok_or_else(|| { - ArrowError::InvalidArgumentError(format!("Size cannot be negative, got {size}")) + let data_type = DataType::FixedSizeBinary(value_length); + let value_size = value_length.to_usize().ok_or_else(|| { + ArrowError::InvalidArgumentError(format!( + "Value length cannot be negative, got {value_length}" + )) })?; - let len = match values.len().checked_div(s) { + let len = match values.len().checked_div(value_size) { Some(len) => { if let Some(n) = nulls.as_ref() { if n.len() != len { @@ -148,40 +154,74 @@ impl FixedSizeBinaryArray { None => { if !values.is_empty() { return Err(ArrowError::InvalidArgumentError( - "Buffer cannot have non-zero length if the item size is zero".to_owned(), + "Buffer cannot have non-zero length if the value length is zero".to_owned(), )); } - // If the item size is zero, try to determine the length from the null buffer + // If the value length is zero, try to determine the length from the null buffer nulls.as_ref().map(|n| n.len()).unwrap_or(0) } }; + Self::validate_lengths(value_size, len)?; + Ok(Self { data_type, value_data: values, - value_length: size, + value_length, nulls, len, }) } + /// Some calculations below use i32 arithmetic which can overflow when + /// valid offsets are past i32::MAX. Until that is solved for real do not + /// permit constructing any FixedSizeBinaryArray that has a valid offset + /// past i32::MAX + fn validate_lengths(value_size: usize, len: usize) -> Result<(), ArrowError> { + // the offset is also calculated for the next element (i + 1) so + // check `len` (not last element index) to ensure that all offsets are valid + let max_offset = value_size.checked_mul(len).ok_or_else(|| { + ArrowError::InvalidArgumentError(format!( + "FixedSizeBinaryArray error: value size {value_size} * len {len} exceeds maximum valid offset" + )) + })?; + + let max_valid_offset: usize = i32::MAX.try_into().map_err(|_| { + ArrowError::InvalidArgumentError(format!( + "FixedSizeBinaryArray error: maximum valid offset exceeds i32::MAX, got {max_offset}" + )) + })?; + + if max_offset > max_valid_offset { + return Err(ArrowError::InvalidArgumentError(format!( + "FixedSizeBinaryArray error: value size {value_size} * length {len} exceeds maximum valid offset of {max_valid_offset}" + ))); + }; + Ok(()) + } + /// Create a new [`FixedSizeBinaryArray`] of length `len` where all values are null /// /// # Panics /// /// Panics if /// - /// * `size < 0` - /// * `size * len` would overflow `usize` - pub fn new_null(size: i32, len: usize) -> Self { + /// * `value_length < 0` + /// * `value_length * len` would overflow `usize` + /// * `value_length * len > i32::MAX` + /// * `value_length * len * 8` would overflow `usize` + pub fn new_null(value_length: i32, len: usize) -> Self { const BITS_IN_A_BYTE: usize = 8; - let capacity_in_bytes = size.to_usize().unwrap().checked_mul(len).unwrap(); + let value_size = value_length.to_usize().unwrap(); + Self::validate_lengths(value_size, len).unwrap(); + let capacity_in_bytes = value_size.checked_mul(len).unwrap(); + let capacity_in_bits = capacity_in_bytes.checked_mul(BITS_IN_A_BYTE).unwrap(); Self { - data_type: DataType::FixedSizeBinary(size), - value_data: MutableBuffer::new_null(capacity_in_bytes * BITS_IN_A_BYTE).into(), + data_type: DataType::FixedSizeBinary(value_length), + value_data: MutableBuffer::new_null(capacity_in_bits).into(), nulls: Some(NullBuffer::new_null(len)), - value_length: size, + value_length, len, } } @@ -315,7 +355,7 @@ impl FixedSizeBinaryArray { U: AsRef<[u8]>, { let mut len = 0; - let mut size = None; + let mut value_size = None; let mut byte = 0; let iter_size_hint = iter.size_hint().0; @@ -333,7 +373,7 @@ impl FixedSizeBinaryArray { if let Some(slice) = item { let slice = slice.as_ref(); - if let Some(size) = size { + if let Some(size) = value_size { if size != slice.len() { return Err(ArrowError::InvalidArgumentError(format!( "Nested array size mismatch: one is {}, and the other is {}", @@ -343,16 +383,24 @@ impl FixedSizeBinaryArray { } } else { let len = slice.len(); - size = Some(len); + value_size = Some(len); // Now that we know how large each element is we can reserve // sufficient capacity in the underlying mutable buffer for // the data. - buffer.reserve(iter_size_hint * len); - buffer.extend_zeros(slice.len() * prepend); + if let Some(capacity) = iter_size_hint.checked_mul(len) { + buffer.reserve(capacity); + } + let prepend_zeros = slice.len().checked_mul(prepend).ok_or_else(|| { + ArrowError::InvalidArgumentError(format!( + "FixedSizeBinaryArray error: value size {} * prepend {prepend} exceeds usize", + slice.len() + )) + })?; + buffer.extend_zeros(prepend_zeros); } bit_util::set_bit(null_buf.as_slice_mut(), len); buffer.extend_from_slice(slice); - } else if let Some(size) = size { + } else if let Some(size) = value_size { buffer.extend_zeros(size); } else { prepend += 1; @@ -371,12 +419,18 @@ impl FixedSizeBinaryArray { let nulls = NullBuffer::from_unsliced_buffer(null_buf, len); - let size = size.unwrap_or(0) as i32; + let value_size = value_size.unwrap_or(0); + Self::validate_lengths(value_size, len)?; + let value_length = value_size.try_into().map_err(|_| { + ArrowError::InvalidArgumentError(format!( + "FixedSizeBinaryArray value length exceeds i32, got {value_size}" + )) + })?; Ok(Self { - data_type: DataType::FixedSizeBinary(size), + data_type: DataType::FixedSizeBinary(value_length), value_data: buffer.into(), nulls, - value_length: size, + value_length, len, }) } @@ -384,8 +438,8 @@ impl FixedSizeBinaryArray { /// Create an array from an iterable argument of sparse byte slices. /// Sparsity means that items returned by the iterator are optional, i.e input argument can /// contain `None` items. In cases where the iterator returns only `None` values, this - /// also takes a size parameter to ensure that the a valid FixedSizeBinaryArray is still - /// created. + /// also takes a `value_length` parameter to ensure that a valid + /// [`FixedSizeBinaryArray`] is still created. /// /// # Examples /// @@ -405,17 +459,30 @@ impl FixedSizeBinaryArray { /// # Errors /// /// Returns error if argument has length zero, or sizes of nested slices don't match. - pub fn try_from_sparse_iter_with_size(mut iter: T, size: i32) -> Result + pub fn try_from_sparse_iter_with_size( + mut iter: T, + value_length: i32, + ) -> Result where T: Iterator>, U: AsRef<[u8]>, { + let value_size = value_length.to_usize().ok_or_else(|| { + ArrowError::InvalidArgumentError(format!( + "Value length cannot be negative, got {value_length}" + )) + })?; let mut len = 0; let mut byte = 0; let iter_size_hint = iter.size_hint().0; let mut null_buf = MutableBuffer::new(bit_util::ceil(iter_size_hint, 8)); - let mut buffer = MutableBuffer::new(iter_size_hint * (size as usize)); + let capacity = iter_size_hint.checked_mul(value_size).ok_or_else(|| { + ArrowError::InvalidArgumentError(format!( + "FixedSizeBinaryArray error: value size {value_size} * len hint {iter_size_hint} exceeds usize" + )) + })?; + let mut buffer = MutableBuffer::new(capacity); iter.try_for_each(|item| -> Result<(), ArrowError> { // extend null bitmask by one byte per each 8 items @@ -427,10 +494,10 @@ impl FixedSizeBinaryArray { if let Some(slice) = item { let slice = slice.as_ref(); - if size as usize != slice.len() { + if value_size != slice.len() { return Err(ArrowError::InvalidArgumentError(format!( "Nested array size mismatch: one is {}, and the other is {}", - size, + value_length, slice.len() ))); } @@ -438,7 +505,7 @@ impl FixedSizeBinaryArray { bit_util::set_bit(null_buf.as_slice_mut(), len); buffer.extend_from_slice(slice); } else { - buffer.extend_zeros(size as usize); + buffer.extend_zeros(value_size); } len += 1; @@ -447,13 +514,14 @@ impl FixedSizeBinaryArray { })?; let nulls = NullBuffer::from_unsliced_buffer(null_buf, len); + Self::validate_lengths(value_size, len)?; Ok(Self { - data_type: DataType::FixedSizeBinary(size), + data_type: DataType::FixedSizeBinary(value_length), value_data: buffer.into(), nulls, len, - value_length: size, + value_length, }) } @@ -480,24 +548,25 @@ impl FixedSizeBinaryArray { U: AsRef<[u8]>, { let mut len = 0; - let mut size = None; + let mut value_size = None; let iter_size_hint = iter.size_hint().0; let mut buffer = MutableBuffer::new(0); iter.try_for_each(|item| -> Result<(), ArrowError> { let slice = item.as_ref(); - if let Some(size) = size { - if size != slice.len() { + if let Some(value_size) = value_size { + if value_size != slice.len() { return Err(ArrowError::InvalidArgumentError(format!( - "Nested array size mismatch: one is {}, and the other is {}", - size, + "Nested array size mismatch: one is {value_size}, and the other is {}", slice.len() ))); } } else { let len = slice.len(); - size = Some(len); - buffer.reserve(iter_size_hint * len); + value_size = Some(len); + if let Some(capacity) = iter_size_hint.checked_mul(len) { + buffer.reserve(capacity); + } } buffer.extend_from_slice(slice); @@ -513,12 +582,18 @@ impl FixedSizeBinaryArray { )); } - let size = size.unwrap_or(0).try_into().unwrap(); + let value_size = value_size.unwrap_or(0); + Self::validate_lengths(value_size, len)?; + let value_length = value_size.try_into().map_err(|_| { + ArrowError::InvalidArgumentError(format!( + "FixedSizeBinaryArray value length exceeds i32, got {value_size}" + )) + })?; Ok(Self { - data_type: DataType::FixedSizeBinary(size), + data_type: DataType::FixedSizeBinary(value_length), value_data: buffer.into(), nulls: None, - value_length: size, + value_length, len, }) } @@ -548,8 +623,15 @@ impl From for FixedSizeBinaryArray { _ => panic!("Expected data type to be FixedSizeBinary"), }; - let size = value_length as usize; - let value_data = buffers[0].slice_with_length(offset * size, len * size); + let value_size = value_length + .to_usize() + .expect("FixedSizeBinaryArray value length must be non-negative"); + Self::validate_lengths(value_size, len) + .expect("FixedSizeBinaryArray offsets must fit within i32"); + let value_data = buffers[0].slice_with_length( + offset.checked_mul(value_size).expect("offset overflow"), + len.checked_mul(value_size).expect("length overflow"), + ); Self { data_type, @@ -1040,6 +1122,26 @@ mod tests { array.value(4); } + #[test] + fn test_validate_lengths_allows_empty_array() { + FixedSizeBinaryArray::validate_lengths(1024, 0).unwrap(); + } + + #[test] + fn test_validate_lengths_allows_i32_max_offset() { + FixedSizeBinaryArray::validate_lengths(1, i32::MAX as usize).unwrap(); + FixedSizeBinaryArray::validate_lengths(262_176, 8191).unwrap(); + } + + #[test] + fn test_validate_lengths_rejects_offset_past_i32_max() { + let err = FixedSizeBinaryArray::validate_lengths(262_177, 8192).unwrap_err(); + assert_eq!( + err.to_string(), + "Invalid argument error: FixedSizeBinaryArray error: value size 262177 * length 8192 exceeds maximum valid offset of 2147483647", + ); + } + #[test] fn test_constructors() { let buffer = Buffer::from_vec(vec![0_u8; 10]); @@ -1063,7 +1165,7 @@ mod tests { assert_eq!( err.to_string(), - "Invalid argument error: Size cannot be negative, got -1" + "Invalid argument error: Value length cannot be negative, got -1" ); let nulls = NullBuffer::new_null(3); @@ -1088,7 +1190,7 @@ mod tests { FixedSizeBinaryArray::try_new(0, buffer, None).unwrap_err(); assert_eq!( zero_sized_with_non_empty_buffer_err.to_string(), - "Invalid argument error: Buffer cannot have non-zero length if the item size is zero" + "Invalid argument error: Buffer cannot have non-zero length if the value length is zero" ); } } diff --git a/arrow-array/src/array/run_array.rs b/arrow-array/src/array/run_array.rs index f317af6a10f0..02bc730b32df 100644 --- a/arrow-array/src/array/run_array.rs +++ b/arrow-array/src/array/run_array.rs @@ -30,6 +30,25 @@ use crate::{ types::{Int16Type, Int32Type, Int64Type, RunEndIndexType}, }; +/// Recursively applies a function to the values of a RunEndEncoded array, preserving the run structure. +/// +/// # Example +/// +/// ```ignore +/// let result = ree_recurse!(array, Int32Type, my_function)?; +/// ``` +/// +/// This macro is useful for implementing functions that should work on the logical values +/// of a REE array while preserving the run-end encoding structure. +#[macro_export] +macro_rules! ree_map { + ($array:expr, $run_type:ty, $func:expr) => {{ + let ree = $array.as_run_opt::<$run_type>().unwrap(); + let inner_values = $func(ree.values().as_ref())?; + Ok(std::sync::Arc::new(ree.with_values(inner_values))) + }}; +} + /// An array of [run-end encoded values]. /// /// This encoding is variation on [run-length encoding (RLE)] and is good for representing @@ -200,6 +219,46 @@ impl RunArray { &self.values } + /// Returns a new [`RunArray`] with the same `run_ends` and the supplied `values`. + /// + /// # Panics + /// + /// Panics if `values.len()` does not equal `self.values().len()`. + /// + /// # Example + /// + /// ``` + /// # use std::sync::Arc; + /// # use arrow_array::{RunArray, Int32Array, StringArray, ArrayRef,Array}; + /// # use arrow_array::types::Int32Type; + /// // A RunArray logically representing ["a", "a", "b", "c", "c"] + /// let run_ends = Int32Array::from(vec![2, 3, 5]); + /// let values: ArrayRef = Arc::new(StringArray::from(vec!["a", "b", "c"])); + /// let run_array = RunArray::::try_new(&run_ends, &values).unwrap(); + /// + /// // Swap in new values while keeping the same run pattern. + /// // The result logically represents ["x", "x", "y", "z", "z"]. + /// let new_values: ArrayRef = Arc::new(StringArray::from(vec!["x", "y", "z"])); + /// let new_run_array = run_array.with_values(new_values); + /// + /// assert_eq!(new_run_array.len(), 5); + /// assert_eq!(new_run_array.run_ends().values(), &[2, 3, 5]); + /// ``` + pub fn with_values(&self, values: ArrayRef) -> Self { + assert_eq!(values.len(), self.values().len()); + let (run_ends_field, values_field) = match &self.data_type { + DataType::RunEndEncoded(r, v) => (r, v), + _ => unreachable!("RunArray should have type RunEndEncoded"), + }; + let data_type = + DataType::RunEndEncoded(Arc::clone(run_ends_field), Arc::clone(values_field)); + Self { + data_type, + run_ends: self.run_ends.clone(), + values, + } + } + /// Similar to [`values`] but accounts for logical slicing, returning only the values /// that are part of the logical slice of this array. /// diff --git a/arrow-array/src/builder/generic_bytes_dictionary_builder.rs b/arrow-array/src/builder/generic_bytes_dictionary_builder.rs index 956014a72b7d..4ce451aec81e 100644 --- a/arrow-array/src/builder/generic_bytes_dictionary_builder.rs +++ b/arrow-array/src/builder/generic_bytes_dictionary_builder.rs @@ -84,7 +84,7 @@ where ) -> Self { Self { state: Default::default(), - dedup: Default::default(), + dedup: HashTable::with_capacity(value_capacity), keys_builder: PrimitiveBuilder::with_capacity(keys_capacity), values_builder: GenericByteBuilder::::with_capacity(value_capacity, data_capacity), } @@ -647,6 +647,25 @@ mod tests { test_bytes_dictionary_builder::>(vec![b"abc", b"def"]); } + #[test] + fn test_with_capacity_presizes_dedup() { + // `with_capacity` must size the dedup `HashTable` from `value_capacity`, + // otherwise the first inserts force a chain of resize+rehash cycles. + let value_capacity = 128; + let builder = + GenericByteDictionaryBuilder::>::with_capacity( + 256, + value_capacity, + value_capacity * 32, + ); + assert!( + builder.dedup.capacity() >= value_capacity, + "dedup HashTable not pre-sized: got capacity {}, expected >= {}", + builder.dedup.capacity(), + value_capacity, + ); + } + fn test_bytes_dictionary_builder_finish_cloned(values: Vec<&T::Native>) where T: ByteArrayType, diff --git a/arrow-cast/src/cast/dictionary.rs b/arrow-cast/src/cast/dictionary.rs index 0092bc0c87dd..601f50a4d001 100644 --- a/arrow-cast/src/cast/dictionary.rs +++ b/arrow-cast/src/cast/dictionary.rs @@ -361,22 +361,18 @@ where D: DecimalType + ArrowPrimitiveType, { let dict = pack_numeric_to_dictionary::(array, dict_value_type, cast_options)?; - let dict = dict - .as_dictionary::() - .downcast_dict::>() - .ok_or_else(|| { - ArrowError::ComputeError(format!( - "Internal Error: Cannot cast dict to {}Array", - D::PREFIX - )) - })?; - let value = dict.values().clone(); - // Set correct precision/scale - let value = value.with_precision_and_scale(precision, scale)?; - Ok(Arc::new(DictionaryArray::::try_new( - dict.keys().clone(), - Arc::new(value), - )?)) + let dict = dict.as_dictionary::(); + let typed = dict.downcast_dict::>().ok_or_else(|| { + ArrowError::ComputeError(format!( + "Internal Error: Cannot cast dict to {}Array", + D::PREFIX + )) + })?; + let value = typed + .values() + .clone() + .with_precision_and_scale(precision, scale)?; + Ok(Arc::new(dict.with_values(Arc::new(value)))) } pub(crate) fn string_view_to_dictionary( diff --git a/arrow-csv/src/reader/records.rs b/arrow-csv/src/reader/records.rs index 33927c93360a..a6d54f867c1d 100644 --- a/arrow-csv/src/reader/records.rs +++ b/arrow-csv/src/reader/records.rs @@ -198,16 +198,21 @@ impl RecordDecoder { // csv_core::Reader writes end offsets relative to the start of the row // Therefore scan through and offset these based on the cumulative row offsets - let mut row_offset = 0; + let mut row_offset: usize = 0; self.offsets[1..self.offsets_len] .chunks_exact_mut(self.num_columns) - .for_each(|row| { + .try_for_each(|row| -> Result<(), ArrowError> { let offset = row_offset; - row.iter_mut().for_each(|x| { - *x += offset; + row.iter_mut().try_for_each(|x| -> Result<(), ArrowError> { + *x = x.checked_add(offset).ok_or_else(|| { + ArrowError::CsvError( + "CSV record offsets overflowed usize while flushing".to_string(), + ) + })?; row_offset = *x; - }); - }); + Ok(()) + }) + })?; // Need to truncate data t1o the actual amount of data read let data = std::str::from_utf8(&self.data[..self.data_len]).map_err(|e| { @@ -399,4 +404,23 @@ mod tests { assert_eq!(read, 5); assert_eq!(bytes, csv.len()); } + + /// Regression test for an overflow path found by the `arrow-csv` + /// cargo-fuzz harness being prototyped for #5332. Stages the + /// `RecordDecoder` state directly so that rebasing the second row's + /// end offset overflows `usize`. With the previous `*x += offset` this + /// panicked with `attempt to add with overflow`; the patched code + /// surfaces the condition as `ArrowError::CsvError`. + #[test] + fn test_flush_offset_overflow_returns_csv_error() { + let mut decoder = RecordDecoder::new(Reader::new(), 1, false); + decoder.offsets = vec![0, usize::MAX, 1]; + decoder.offsets_len = 3; + decoder.num_rows = 2; + let err = decoder.flush().unwrap_err(); + assert_eq!( + err.to_string(), + "Csv error: CSV record offsets overflowed usize while flushing" + ); + } } diff --git a/arrow-flight/src/client.rs b/arrow-flight/src/client.rs index dac086271cb7..b2059a81d0df 100644 --- a/arrow-flight/src/client.rs +++ b/arrow-flight/src/client.rs @@ -31,6 +31,7 @@ use futures::{ stream::{self, BoxStream}, }; use prost::Message; +use tonic::codegen::{Body, StdError}; use tonic::{metadata::MetadataMap, transport::Channel}; use crate::error::{FlightError, Result}; @@ -67,22 +68,28 @@ use crate::streams::{FallibleRequestStream, FallibleTonicResponseStream}; /// # } /// ``` #[derive(Debug)] -pub struct FlightClient { +pub struct FlightClient { /// Optional grpc header metadata to include with each request metadata: MetadataMap, /// The inner client - inner: FlightServiceClient, + inner: FlightServiceClient, } -impl FlightClient { - /// Creates a client client with the provided [`Channel`] - pub fn new(channel: Channel) -> Self { - Self::new_from_inner(FlightServiceClient::new(channel)) +impl FlightClient +where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, +{ + /// Creates a client with the provided transport + pub fn new(inner: T) -> Self { + Self::new_from_inner(FlightServiceClient::new(inner)) } /// Creates a new higher level client with the provided lower level client - pub fn new_from_inner(inner: FlightServiceClient) -> Self { + pub fn new_from_inner(inner: FlightServiceClient) -> Self { Self { metadata: MetadataMap::new(), inner, @@ -120,19 +127,19 @@ impl FlightClient { /// Return a reference to the underlying tonic /// [`FlightServiceClient`] - pub fn inner(&self) -> &FlightServiceClient { + pub fn inner(&self) -> &FlightServiceClient { &self.inner } /// Return a mutable reference to the underlying tonic /// [`FlightServiceClient`] - pub fn inner_mut(&mut self) -> &mut FlightServiceClient { + pub fn inner_mut(&mut self) -> &mut FlightServiceClient { &mut self.inner } /// Consume this client and return the underlying tonic /// [`FlightServiceClient`] - pub fn into_inner(self) -> FlightServiceClient { + pub fn into_inner(self) -> FlightServiceClient { self.inner } @@ -664,10 +671,272 @@ impl FlightClient { } /// return a Request, adding any configured metadata - fn make_request(&self, t: T) -> tonic::Request { + fn make_request(&self, t: R) -> tonic::Request { // Pass along metadata let mut request = tonic::Request::new(t); *request.metadata_mut() = self.metadata.clone(); request } } + +#[cfg(test)] +mod tests { + use super::FlightClient; + use crate::encode::FlightDataEncoderBuilder; + use crate::flight_service_server::{FlightService, FlightServiceServer}; + use crate::{ + Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightInfo, + HandshakeRequest, HandshakeResponse, PollInfo, PutResult, SchemaResult, Ticket, + }; + use arrow_array::{RecordBatch, UInt64Array}; + use bytes::Bytes; + use futures::{StreamExt, TryStreamExt, stream::BoxStream}; + use std::net::SocketAddr; + use std::sync::{Arc, Mutex}; + use std::time::Duration; + use tokio::net::TcpListener; + use tokio::task::JoinHandle; + use tonic::metadata::MetadataMap; + use tonic::service::interceptor::InterceptedService; + use tonic::transport::Channel; + use tonic::{Request, Response, Status, Streaming}; + use uuid::Uuid; + + /// Minimal `FlightService` that records request metadata and serves a + /// configured `do_get` response. Other RPCs return `Unimplemented`. + #[derive(Debug, Clone, Default)] + struct InterceptorTestServer { + state: Arc>, + } + + #[derive(Debug, Default)] + struct InterceptorTestState { + do_get_request: Option, + do_get_response: Option>>, + last_request_metadata: Option, + } + + impl InterceptorTestServer { + fn save_metadata(&self, request: &Request) { + self.state.lock().unwrap().last_request_metadata = Some(request.metadata().clone()); + } + + fn set_do_get_response(&self, response: Vec>) { + self.state.lock().unwrap().do_get_response = Some(response); + } + + fn take_do_get_request(&self) -> Option { + self.state.lock().unwrap().do_get_request.take() + } + + fn take_last_request_metadata(&self) -> Option { + self.state.lock().unwrap().last_request_metadata.take() + } + } + + #[tonic::async_trait] + impl FlightService for InterceptorTestServer { + type HandshakeStream = BoxStream<'static, Result>; + type ListFlightsStream = BoxStream<'static, Result>; + type DoGetStream = BoxStream<'static, Result>; + type DoPutStream = BoxStream<'static, Result>; + type DoActionStream = BoxStream<'static, Result>; + type ListActionsStream = BoxStream<'static, Result>; + type DoExchangeStream = BoxStream<'static, Result>; + + async fn do_get( + &self, + request: Request, + ) -> Result, Status> { + self.save_metadata(&request); + let mut state = self.state.lock().unwrap(); + state.do_get_request = Some(request.into_inner()); + + let batches = state + .do_get_response + .take() + .ok_or_else(|| Status::internal("no do_get response configured"))?; + let batch_stream = futures::stream::iter(batches).map_err(Into::into); + let stream = FlightDataEncoderBuilder::new() + .build(batch_stream) + .map_err(Into::into); + Ok(Response::new(stream.boxed())) + } + + async fn handshake( + &self, + _: Request>, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + async fn list_flights( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + async fn get_flight_info( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + async fn poll_flight_info( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + async fn get_schema( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + async fn do_put( + &self, + _: Request>, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + async fn do_action( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + async fn list_actions( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + async fn do_exchange( + &self, + _: Request>, + ) -> Result, Status> { + Err(Status::unimplemented("")) + } + } + + /// Spawns the test server on a background task and exposes a connected channel. + struct InterceptorTestFixture { + shutdown: Option>, + addr: SocketAddr, + handle: Option>>, + } + + impl InterceptorTestFixture { + async fn new(server: FlightServiceServer) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = tokio::sync::oneshot::channel(); + let shutdown_future = async move { + rx.await.ok(); + }; + let serve = tonic::transport::Server::builder() + .timeout(Duration::from_secs(30)) + .add_service(server) + .serve_with_incoming_shutdown( + tokio_stream::wrappers::TcpListenerStream::new(listener), + shutdown_future, + ); + let handle = tokio::task::spawn(serve); + Self { + shutdown: Some(tx), + addr, + handle: Some(handle), + } + } + + async fn channel(&self) -> Channel { + let url = format!("http://{}", self.addr); + tonic::transport::Endpoint::from_shared(url) + .expect("valid endpoint") + .timeout(Duration::from_secs(30)) + .connect() + .await + .expect("error connecting to server") + } + + async fn shutdown_and_wait(mut self) { + if let Some(tx) = self.shutdown.take() { + tx.send(()).expect("server quit early"); + } + if let Some(handle) = self.handle.take() { + handle + .await + .expect("task join error (panic?)") + .expect("server error at shutdown"); + } + } + } + + /// Integration test: a tonic [`Channel`] wrapped in an [`InterceptedService`] + /// that injects a custom header is passed to [`FlightClient`], and the server + /// observes the header on the request. + #[tokio::test] + async fn test_flight_client_with_intercepted_channel_passes_custom_header() { + let test_server = InterceptorTestServer::default(); + let fixture = + InterceptorTestFixture::new(FlightServiceServer::new(test_server.clone())).await; + + let channel = fixture.channel().await; + + let header_name = "x-random-header"; + let header_value = format!("random-{}", Uuid::new_v4()); + let header_value_for_interceptor = header_value.clone(); + + let interceptor = move |mut req: Request<()>| -> Result, Status> { + req.metadata_mut().insert( + header_name, + header_value_for_interceptor + .parse() + .expect("valid metadata value"), + ); + Ok(req) + }; + + let intercepted = InterceptedService::new(channel, interceptor); + let mut client = FlightClient::new(intercepted); + + let ticket = Ticket { + ticket: Bytes::from("dummy-ticket"), + }; + + let batch = RecordBatch::try_from_iter(vec![( + "col", + Arc::new(UInt64Array::from_iter([1, 2, 3, 4])) as _, + )]) + .unwrap(); + + test_server.set_do_get_response(vec![Ok(batch.clone())]); + + let response_stream = client + .do_get(ticket.clone()) + .await + .expect("error making do_get request"); + + let response: Vec = response_stream + .try_collect() + .await + .expect("error streaming data"); + + assert_eq!(response, vec![batch]); + assert_eq!(test_server.take_do_get_request(), Some(ticket)); + + let metadata = test_server + .take_last_request_metadata() + .expect("server received headers") + .into_headers(); + + let received = metadata + .get(header_name) + .expect("interceptor header missing on server") + .to_str() + .expect("ascii header value"); + assert_eq!(received, header_value); + + fixture.shutdown_and_wait().await; + } +} diff --git a/arrow-ipc/src/tests/delta_dictionary.rs b/arrow-ipc/src/tests/delta_dictionary.rs index dfd8cd33e550..60a6c945a3be 100644 --- a/arrow-ipc/src/tests/delta_dictionary.rs +++ b/arrow-ipc/src/tests/delta_dictionary.rs @@ -24,9 +24,11 @@ use crate::{ writer::FileWriter, }; use arrow_array::{ - Array, ArrayRef, DictionaryArray, RecordBatch, StringArray, builder::StringDictionaryBuilder, + Array, ArrayRef, DictionaryArray, ListArray, RecordBatch, StringArray, StructArray, + builder::{ArrayBuilder, ListBuilder, StringDictionaryBuilder, StructBuilder}, types::Int32Type, }; + use arrow_schema::{DataType, Field, Schema}; use std::io::Cursor; use std::sync::Arc; @@ -35,7 +37,7 @@ use std::sync::Arc; fn test_zero_row_dict() { let batches: &[&[&str]] = &[&[], &["A"], &[], &["B", "C"], &[]]; run_delta_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(vec![]), MessageType::RecordBatch, @@ -48,7 +50,7 @@ fn test_zero_row_dict() { ); run_resend_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(vec![]), MessageType::RecordBatch, @@ -72,7 +74,7 @@ fn test_mixed_delta() { ]; run_delta_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A"])), MessageType::RecordBatch, @@ -87,7 +89,7 @@ fn test_mixed_delta() { ); run_resend_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A"])), MessageType::RecordBatch, @@ -106,7 +108,7 @@ fn test_mixed_delta() { fn test_disjoint_delta() { let batches: &[&[&str]] = &[&["A"], &["B"], &["C", "E"]]; run_delta_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A"])), MessageType::RecordBatch, @@ -118,7 +120,7 @@ fn test_disjoint_delta() { ); run_resend_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A"])), MessageType::RecordBatch, @@ -134,7 +136,7 @@ fn test_disjoint_delta() { fn test_increasing_delta() { let batches: &[&[&str]] = &[&["A"], &["A", "B"], &["A", "B", "C"]]; run_delta_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A"])), MessageType::RecordBatch, @@ -146,7 +148,7 @@ fn test_increasing_delta() { ); run_resend_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A"])), MessageType::RecordBatch, @@ -162,7 +164,7 @@ fn test_increasing_delta() { fn test_single_delta() { let batches: &[&[&str]] = &[&["A", "B", "C"], &["D"]]; run_delta_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A", "B", "C"])), MessageType::RecordBatch, @@ -172,7 +174,7 @@ fn test_single_delta() { ); run_resend_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A", "B", "C"])), MessageType::RecordBatch, @@ -186,7 +188,7 @@ fn test_single_delta() { fn test_single_same_value_sequence() { let batches: &[&[&str]] = &[&["A"], &["A"], &["A"], &["A"]]; run_delta_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A"])), MessageType::RecordBatch, @@ -197,7 +199,7 @@ fn test_single_same_value_sequence() { ); run_resend_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A"])), MessageType::RecordBatch, @@ -216,7 +218,7 @@ fn str_vec(strings: &[&str]) -> Vec { fn test_multi_same_value_sequence() { let batches: &[&[&str]] = &[&["A", "B", "C"], &["A", "B", "C"]]; run_delta_sequence_test( - batches, + &build_batches(batches), &[ MessageType::Dict(str_vec(&["A", "B", "C"])), MessageType::RecordBatch, @@ -232,17 +234,17 @@ enum MessageType { RecordBatch, } -fn run_resend_sequence_test(batches: &[&[&str]], sequence: &[MessageType]) { +fn run_resend_sequence_test(batches: &[RecordBatch], sequence: &[MessageType]) { let opts = IpcWriteOptions::default().with_dictionary_handling(DictionaryHandling::Resend); run_sequence_test(batches, sequence, opts); } -fn run_delta_sequence_test(batches: &[&[&str]], sequence: &[MessageType]) { +fn run_delta_sequence_test(batches: &[RecordBatch], sequence: &[MessageType]) { let opts = IpcWriteOptions::default().with_dictionary_handling(DictionaryHandling::Delta); run_sequence_test(batches, sequence, opts); } -fn run_sequence_test(batches: &[&[&str]], sequence: &[MessageType], options: IpcWriteOptions) { +fn run_sequence_test(batches: &[RecordBatch], sequence: &[MessageType], options: IpcWriteOptions) { let stream_buf = write_all_to_stream(options.clone(), batches); let ipc_stream = get_ipc_message_stream(stream_buf); for (message, expected) in ipc_stream.iter().zip(sequence.iter()) { @@ -310,7 +312,7 @@ fn test_replace_same_length() { &["A", "B", "C", "D", "E", "F"], &["A", "G", "H", "I", "J", "K"], ]; - run_parity_test(batches); + run_parity_test(&build_batches(batches)); } #[test] @@ -323,14 +325,14 @@ fn test_sparse_deltas() { &["parquet", "B"], &["123", "B", "C"], ]; - run_parity_test(batches); + run_parity_test(&build_batches(batches)); } #[test] fn test_deltas_with_reset() { // Dictionary resets at ["C", "D"] let batches: &[&[&str]] = &[&["A"], &["A", "B"], &["C", "D"], &["A", "B", "C", "D"]]; - run_parity_test(batches); + run_parity_test(&build_batches(batches)); } /// FileWriter can only tolerate very specific patterns of delta dictionaries, @@ -338,7 +340,19 @@ fn test_deltas_with_reset() { #[test] fn test_deltas_with_file() { let batches: &[&[&str]] = &[&["A"], &["A", "B"], &["A", "B", "C"], &["A", "B", "C", "D"]]; - run_parity_test(batches); + run_parity_test(&build_batches(batches)); +} + +#[test] +fn test_deltas_with_in_struct() { + let batches: &[&[&str]] = &[&["A"], &["A", "B"], &["A", "B", "C"], &["A", "B", "C", "D"]]; + run_parity_test(&build_struct_batches(batches)); +} + +#[test] +fn test_deltas_with_in_list() { + let batches: &[&[&str]] = &[&["A"], &["A", "B"], &["A", "B", "C"], &["A", "B", "C", "D"]]; + run_parity_test(&build_list_batches(batches)); } /// Encode all batches three times and compare all three for the same results @@ -348,7 +362,7 @@ fn test_deltas_with_file() { /// - Stream encoding without delta /// - File encoding with delta (File format does not allow replacement /// dictionaries) -fn run_parity_test(batches: &[&[&str]]) { +fn run_parity_test(batches: &[RecordBatch]) { let delta_options = IpcWriteOptions::default().with_dictionary_handling(DictionaryHandling::Delta); let delta_stream_buf = write_all_to_stream(delta_options.clone(), batches); @@ -368,16 +382,16 @@ fn run_parity_test(batches: &[&[&str]]) { let (first_stream, other_streams) = streams.split_first_mut().unwrap(); for (idx, batch) in first_stream.by_ref().enumerate() { - let first_dict = extract_dictionary(batch); - let expected_values = batches[idx]; - assert_eq!(expected_values, &dict_to_vec(first_dict.clone())); + let first_dict = extract_dictionary(&batch); + let expected_values = dict_to_vec(&extract_dictionary(&batches[idx])); + assert_eq!(expected_values, dict_to_vec(&first_dict)); for stream in other_streams.iter_mut() { let next_batch = stream .next() .expect("All streams should yield same number of elements"); - let next_dict = extract_dictionary(next_batch); - assert_eq!(expected_values, &dict_to_vec(next_dict.clone())); + let next_dict = extract_dictionary(&next_batch); + assert_eq!(expected_values, dict_to_vec(&next_dict)); assert_eq!(first_dict, next_dict); } } @@ -390,7 +404,7 @@ fn run_parity_test(batches: &[&[&str]]) { } } -fn dict_to_vec(dict: DictionaryArray) -> Vec { +fn dict_to_vec(dict: &DictionaryArray) -> Vec { dict.downcast_dict::() .unwrap() .into_iter() @@ -418,35 +432,43 @@ fn get_file_batches(buf: Vec) -> Box> { ) } -fn extract_dictionary(batch: RecordBatch) -> DictionaryArray { - batch - .column(0) +fn extract_dictionary(batch: &RecordBatch) -> DictionaryArray { + let mut column = batch.column(0); + + // if we've been passed a struct, assume the first column contains the dict + if let Some(struct_arr) = column.as_any().downcast_ref::() { + column = struct_arr.column(0); + } + + // if we've been passed a list, assume the lists' values are the dict + if let Some(list_arr) = column.as_any().downcast_ref::() { + column = list_arr.values(); + } + + column .as_any() .downcast_ref::>() .unwrap() .clone() } -fn write_all_to_file(options: IpcWriteOptions, vals: &[&[&str]]) -> Vec { - let batches = build_batches(vals); +fn write_all_to_file(options: IpcWriteOptions, batches: &[RecordBatch]) -> Vec { let mut buf: Vec = Vec::new(); let mut writer = FileWriter::try_new_with_options(&mut buf, &batches[0].schema(), options).unwrap(); for batch in batches { - writer.write(&batch).unwrap(); + writer.write(batch).unwrap(); } writer.finish().unwrap(); buf } -fn write_all_to_stream(options: IpcWriteOptions, vals: &[&[&str]]) -> Vec { - let batches = build_batches(vals); - +fn write_all_to_stream(options: IpcWriteOptions, batches: &[RecordBatch]) -> Vec { let mut buf: Vec = Vec::new(); let mut writer = StreamWriter::try_new_with_options(&mut buf, &batches[0].schema(), options).unwrap(); for batch in batches { - writer.write(&batch).unwrap(); + writer.write(batch).unwrap(); } writer.finish().unwrap(); @@ -477,3 +499,71 @@ fn build_batch( RecordBatch::try_new(schema.clone(), vec![Arc::new(array) as ArrayRef]).unwrap() } + +/// build batches where the dictionary array is nested within a struct array. The dictionary array +/// is the first field within the struct. +fn build_struct_batches(vals: &[&[&str]]) -> Vec { + let total_vals = vals.iter().map(|v| v.len()).sum(); + let mut struct_builder = StructBuilder::from_fields( + vec![Field::new( + "struct", + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + false, + )], + total_vals, + ); + + vals.iter() + .map(|v| build_struct_batch(v, &mut struct_builder)) + .collect() +} + +fn build_struct_batch(vals: &[&str], struct_builder: &mut StructBuilder) -> RecordBatch { + for &val in vals { + let dict_builder = struct_builder + .field_builder::>(0) + .unwrap(); + dict_builder.append_value(val); + struct_builder.append(true); + } + + let array = struct_builder.finish_preserve_values(); + + let schema = Arc::new(Schema::new(vec![Field::new( + "dict", + array.data_type().clone(), + true, + )])); + + RecordBatch::try_new(schema.clone(), vec![Arc::new(array) as ArrayRef]).unwrap() +} + +/// builds batches where the dictionary array is nested within a list array +fn build_list_batches(vals: &[&[&str]]) -> Vec { + let mut list_builder = ListBuilder::new(StringDictionaryBuilder::::new()); + + vals.iter() + .map(|v| build_list_batch(v, &mut list_builder)) + .collect() +} + +fn build_list_batch( + vals: &[&str], + list_builder: &mut ListBuilder>, +) -> RecordBatch { + for &val in vals { + let vals_builder = list_builder.values(); + vals_builder.append(val).unwrap(); + list_builder.append(true); + } + + let array = list_builder.finish_preserve_values(); + + let schema = Arc::new(Schema::new(vec![Field::new( + "dict", + array.data_type().clone(), + true, + )])); + + RecordBatch::try_new(schema.clone(), vec![Arc::new(array) as ArrayRef]).unwrap() +} diff --git a/arrow-ipc/src/writer.rs b/arrow-ipc/src/writer.rs index a05072a2c47c..46e2dd7739e0 100644 --- a/arrow-ipc/src/writer.rs +++ b/arrow-ipc/src/writer.rs @@ -1351,7 +1351,7 @@ impl RecordBatchWriter for FileWriter { /// // You must set `.with_dictionary_handling(DictionaryHandling::Delta)` to /// // enable delta dictionaries in the writer /// let options = IpcWriteOptions::default().with_dictionary_handling(DictionaryHandling::Delta); -/// let mut writer = StreamWriter::try_new(&mut stream, &schema).unwrap(); +/// let mut writer = StreamWriter::try_new_with_options(&mut stream, &schema, options).unwrap(); /// /// // When writing the first batch, a dictionary message with 'a' and 'b' will be written /// // prior to the record batch. diff --git a/arrow-select/src/interleave.rs b/arrow-select/src/interleave.rs index f5904bc171ee..1bd34a7f3a58 100644 --- a/arrow-select/src/interleave.rs +++ b/arrow-select/src/interleave.rs @@ -108,6 +108,12 @@ pub fn interleave( DataType::Struct(fields) => interleave_struct(fields, values, indices), DataType::List(field) => interleave_list::(values, indices, field), DataType::LargeList(field) => interleave_list::(values, indices, field), + DataType::RunEndEncoded(r, _) => match r.data_type() { + DataType::Int16 => interleave_run_end::(values, indices), + DataType::Int32 => interleave_run_end::(values, indices), + DataType::Int64 => interleave_run_end::(values, indices), + t => unreachable!("illegal run-end type {t}"), + }, _ => interleave_fallback(values, indices) } } @@ -411,6 +417,71 @@ fn interleave_list( Ok(Arc::new(list_array)) } +/// Specialized [`interleave`] for [`RunArray`]. +fn interleave_run_end( + values: &[&dyn Array], + indices: &[(usize, usize)], +) -> Result { + if indices.is_empty() { + return Ok(new_empty_array(values[0].data_type())); + } + + let n = indices.len(); + R::Native::from_usize(n).ok_or_else(|| { + ArrowError::ComputeError(format!( + "interleave_run_end: output length {n} does not fit run-end type" + )) + })?; + + let runs: Vec<&RunArray> = values.iter().map(|a| a.as_run::()).collect(); + let value_arrays: Vec<&dyn Array> = runs.iter().map(|r| r.values().as_ref()).collect(); + + // Resolve each (array, logical_row) to (array, physical_row), so we can + // lookup physical indices by batch. + let mut phys_pairs: Vec<(usize, usize)> = vec![(0, 0); n]; + let mut grouped: Vec<(Vec, Vec)> = + (0..runs.len()).map(|_| (Vec::new(), Vec::new())).collect(); + for (out_pos, &(arr, row)) in indices.iter().enumerate() { + let row = R::Native::from_usize(row).ok_or_else(|| { + ArrowError::InvalidArgumentError(format!( + "interleave_run_end: row index {row} not representable as run-end type {}", + R::DATA_TYPE + )) + })?; + grouped[arr].0.push(row); + grouped[arr].1.push(out_pos); + } + for (arr_idx, (logical_rows, out_positions)) in grouped.into_iter().enumerate() { + let phys = runs[arr_idx].get_physical_indices(&logical_rows)?; + for (p, out_pos) in phys.iter().zip(out_positions.iter()) { + phys_pairs[*out_pos] = (arr_idx, *p); + } + } + + // Coalesce by physical-pair equality only: emit a new run when the + // (array_idx, physical_idx) pair changes between adjacent output rows. + // TODO: We could perform an equality check across sources to extend the + // output run, but we can't call make_comparator from this crate. + let mut run_ends_buf: Vec = Vec::with_capacity(n); + let mut dedup_pairs: Vec<(usize, usize)> = Vec::with_capacity(n); + dedup_pairs.push(phys_pairs[0]); + for i in 1..n { + if phys_pairs[i] != phys_pairs[i - 1] { + run_ends_buf.push(R::Native::from_usize(i).unwrap()); + dedup_pairs.push(phys_pairs[i]); + } + } + run_ends_buf.push(R::Native::from_usize(n).unwrap()); + + let taken_values = interleave(&value_arrays, &dedup_pairs)?; + let run_ends = PrimitiveArray::::from_iter_values(run_ends_buf); + + Ok(Arc::new(RunArray::::try_new( + &run_ends, + taken_values.as_ref(), + )?)) +} + /// Fallback implementation of interleave using [`MutableArrayData`] fn interleave_fallback( values: &[&dyn Array], diff --git a/arrow-string/src/concat_elements.rs b/arrow-string/src/concat_elements.rs index 41be8a81cb12..cd4676d28752 100644 --- a/arrow-string/src/concat_elements.rs +++ b/arrow-string/src/concat_elements.rs @@ -18,10 +18,12 @@ //! Provides utility functions for concatenation of elements in arrays. use std::sync::Arc; -use arrow_array::builder::BufferBuilder; +use arrow_array::builder::{ + BinaryViewBuilder, BufferBuilder, FixedSizeBinaryBuilder, StringViewBuilder, +}; use arrow_array::types::ByteArrayType; use arrow_array::*; -use arrow_buffer::{ArrowNativeType, NullBuffer}; +use arrow_buffer::{ArrowNativeType, MutableBuffer, NullBuffer}; use arrow_data::ArrayDataBuilder; use arrow_schema::{ArrowError, DataType}; @@ -168,6 +170,129 @@ pub fn concat_elements_utf8_many( Ok(unsafe { builder.build_unchecked() }.into()) } +/// Returns the elementwise concatenation of a [`FixedSizeBinaryArray`]. +/// +/// The result has `value_length = left.value_length() + right.value_length()`. +/// An index is null if either input is null at that position. +/// +/// An error will be returned if `left` and `right` have different lengths. +pub fn concat_elements_fixed_size_binary( + left: &FixedSizeBinaryArray, + right: &FixedSizeBinaryArray, +) -> Result { + if left.len() != right.len() { + return Err(ArrowError::ComputeError(format!( + "Arrays must have the same length: {} != {}", + left.len(), + right.len() + ))); + } + + let left_size = left.value_length() as usize; + let right_size = right.value_length() as usize; + let output_size = left_size + right_size; + + // Pre-compute combined null bitmap so the per-row NULL check is efficient + let nulls = NullBuffer::union(left.nulls(), right.nulls()); + + let mut result = FixedSizeBinaryBuilder::with_capacity(left.len(), output_size as i32); + let mut buffer = MutableBuffer::with_capacity(output_size); + for i in 0..left.len() { + if nulls.as_ref().is_some_and(|n| n.is_null(i)) { + result.append_null(); + } else { + buffer.clear(); + buffer.extend_from_slice(left.value(i)); + buffer.extend_from_slice(right.value(i)); + result.append_value(&buffer)?; + } + } + + Ok(result.finish()) +} + +/// Concatenates two `BinaryViewArray`s element-wise. +/// If either element is `Null`, the result element is also `Null`. +/// +/// # Errors +/// - Returns an error if the input arrays have different lengths. +/// - Returns an error if any concatenated value exceeds `u32::MAX` in length. +pub fn concat_elements_binary_view_array( + left: &BinaryViewArray, + right: &BinaryViewArray, +) -> Result { + if left.len() != right.len() { + return Err(ArrowError::ComputeError(format!( + "Arrays must have the same length: {} != {}", + left.len(), + right.len() + ))); + } + let mut result = BinaryViewBuilder::with_capacity(left.len()); + + // Avoid reallocations by writing to a reused buffer + let mut buffer = MutableBuffer::new(0); + + // Pre-compute combined null bitmap, so the per-row NULL check is efficient + let nulls = NullBuffer::union(left.nulls(), right.nulls()); + + for i in 0..left.len() { + if nulls.as_ref().is_some_and(|n| n.is_null(i)) { + result.append_null(); + } else { + buffer.clear(); + buffer.extend_from_slice(left.value(i)); + buffer.extend_from_slice(right.value(i)); + result.try_append_value(&buffer)?; + } + } + Ok(result.finish()) +} + +/// Concatenates two `StringViewArray`s element-wise. +/// If either element is `Null`, the result element is also `Null`. +/// +/// # Errors +/// - Returns an error if the input arrays have different lengths. +/// - Returns an error if any concatenated value exceeds `u32::MAX` in length. +/// - Returns an error if concatenated strings do not result in a proper UTF-8 string +// Cannot reuse code with `GenericByteViewBuilder` since `try_append_value` works with +// `AsRef`, and there is no conversion from `ByteViewType` to this or [u8] +pub fn concat_elements_string_view_array( + left: &StringViewArray, + right: &StringViewArray, +) -> Result { + if left.len() != right.len() { + return Err(ArrowError::ComputeError(format!( + "Arrays must have the same length: {} != {}", + left.len(), + right.len() + ))); + } + + let mut result = StringViewBuilder::with_capacity(left.len()); + + // Avoid reallocations by writing to a reused buffer + let mut buffer: Vec = Vec::new(); + + let nulls = NullBuffer::union(left.nulls(), right.nulls()); + + for i in 0..left.len() { + if nulls.as_ref().is_some_and(|n| n.is_null(i)) { + result.append_null(); + } else { + buffer.clear(); + buffer.extend_from_slice(left.value(i).as_bytes()); + buffer.extend_from_slice(right.value(i).as_bytes()); + let s = std::str::from_utf8(&buffer).map_err(|_| { + ArrowError::ComputeError("Concatenated values are not valid UTF-8".into()) + })?; + result.try_append_value(s)?; + } + } + Ok(result.finish()) +} + /// Returns the elementwise concatenation of [`Array`]s. /// /// # Errors @@ -185,22 +310,43 @@ pub fn concat_elements_dyn(left: &dyn Array, right: &dyn Array) -> Result { let left = left.as_any().downcast_ref::().unwrap(); let right = right.as_any().downcast_ref::().unwrap(); - Ok(Arc::new(concat_elements_utf8(left, right).unwrap())) + Ok(Arc::new(concat_elements_utf8(left, right)?)) } (DataType::LargeUtf8, DataType::LargeUtf8) => { let left = left.as_any().downcast_ref::().unwrap(); let right = right.as_any().downcast_ref::().unwrap(); - Ok(Arc::new(concat_elements_utf8(left, right).unwrap())) + Ok(Arc::new(concat_elements_utf8(left, right)?)) } (DataType::Binary, DataType::Binary) => { let left = left.as_any().downcast_ref::().unwrap(); let right = right.as_any().downcast_ref::().unwrap(); - Ok(Arc::new(concat_element_binary(left, right).unwrap())) + Ok(Arc::new(concat_element_binary(left, right)?)) } (DataType::LargeBinary, DataType::LargeBinary) => { let left = left.as_any().downcast_ref::().unwrap(); let right = right.as_any().downcast_ref::().unwrap(); - Ok(Arc::new(concat_element_binary(left, right).unwrap())) + Ok(Arc::new(concat_element_binary(left, right)?)) + } + (DataType::BinaryView, DataType::BinaryView) => { + let left = left.as_any().downcast_ref::().unwrap(); + let right = right.as_any().downcast_ref::().unwrap(); + Ok(Arc::new(concat_elements_binary_view_array(left, right)?)) + } + (DataType::Utf8View, DataType::Utf8View) => { + let left = left.as_any().downcast_ref::().unwrap(); + let right = right.as_any().downcast_ref::().unwrap(); + Ok(Arc::new(concat_elements_string_view_array(left, right)?)) + } + (DataType::FixedSizeBinary(_), DataType::FixedSizeBinary(_)) => { + let left = left + .as_any() + .downcast_ref::() + .unwrap(); + let right = right + .as_any() + .downcast_ref::() + .unwrap(); + Ok(Arc::new(concat_elements_fixed_size_binary(left, right)?)) } // unimplemented _ => Err(ArrowError::NotYetImplemented(format!( @@ -213,6 +359,8 @@ pub fn concat_elements_dyn(left: &dyn Array, right: &dyn Array) -> Result>); + let right = BinaryViewArray::from_iter(vec![] as Vec>); + + let output = concat_elements_binary_view_array(&left, &right).unwrap(); + let expected = BinaryViewArray::from_iter(vec![] as Vec>); + assert_eq!(output, expected); + } + #[test] fn test_concat_dyn_same_type() { // test for StringArray @@ -398,6 +661,26 @@ mod tests { .into(); let expected = LargeBinaryArray::from_opt_vec(vec![None, Some(b"baryyy"), None]); assert_eq!(output, expected); + + // test for BinaryViewArray + let left = BinaryViewArray::from_iter(vec![Some(b"foo" as &[u8]), Some(b"bar"), None]); + let right = BinaryViewArray::from_iter(vec![None, Some(b"yyy" as &[u8]), Some(b"zzz")]); + let output: BinaryViewArray = concat_elements_dyn(&left, &right) + .unwrap() + .into_data() + .into(); + let expected = BinaryViewArray::from_iter(vec![None, Some(b"baryyy" as &[u8]), None]); + assert_eq!(output, expected); + + // test for FixedSizeBinaryArray + let left = FixedSizeBinaryArray::from(vec![Some(b"foo" as &[u8]), Some(b"bar"), None]); + let right = FixedSizeBinaryArray::from(vec![None, Some(b"yyy" as &[u8]), Some(b"zzz")]); + let output: FixedSizeBinaryArray = concat_elements_dyn(&left, &right) + .unwrap() + .into_data() + .into(); + let expected = FixedSizeBinaryArray::from(vec![None, Some(b"baryyy" as &[u8]), None]); + assert_eq!(output, expected); } #[test] diff --git a/arrow-string/src/length.rs b/arrow-string/src/length.rs index ff98c5632b74..feefe1247e2c 100644 --- a/arrow-string/src/length.rs +++ b/arrow-string/src/length.rs @@ -17,25 +17,12 @@ //! Defines kernel for length of string arrays and binary arrays +use arrow_array::ree_map; use arrow_array::*; use arrow_array::{cast::AsArray, types::*}; use arrow_buffer::{ArrowNativeType, NullBuffer, OffsetBuffer}; use arrow_schema::{ArrowError, DataType}; use std::sync::Arc; -macro_rules! ree_length { - ($array:expr, $run_type:ty, $k:expr, $v:expr) => {{ - let ree = $array.as_run_opt::<$run_type>().unwrap(); - let inner_value_lengths = length(ree.values().as_ref())?; - let out_ree = unsafe { - RunArray::<$run_type>::new_unchecked( - DataType::RunEndEncoded(Arc::clone($k), Arc::clone($v)), - ree.run_ends().clone(), - inner_value_lengths, - ) - }; - Ok(Arc::new(out_ree) as ArrayRef) - }}; -} fn length_impl( offsets: &OffsetBuffer, @@ -130,10 +117,10 @@ pub fn length(array: &dyn Array) -> Result { list.nulls().cloned(), )?)) } - DataType::RunEndEncoded(k, v) => match k.data_type() { - DataType::Int16 => ree_length!(array, Int16Type, &k, &v), - DataType::Int32 => ree_length!(array, Int32Type, &k, &v), - DataType::Int64 => ree_length!(array, Int64Type, &k, &v), + DataType::RunEndEncoded(k, _) => match k.data_type() { + DataType::Int16 => ree_map!(array, Int16Type, length), + DataType::Int32 => ree_map!(array, Int32Type, length), + DataType::Int64 => ree_map!(array, Int64Type, length), _ => Err(ArrowError::InvalidArgumentError(format!( "Invalid run-end type: {:?}", k.data_type() @@ -149,7 +136,7 @@ pub fn length(array: &dyn Array) -> Result { /// /// * this only accepts StringArray/Utf8, LargeString/LargeUtf8, StringViewArray/Utf8View, /// BinaryArray, LargeBinaryArray, BinaryViewArray, and FixedSizeBinaryArray, -/// or DictionaryArray with above Arrays as values +/// or DictionaryArray/REE with above Arrays as values /// * bit_length of null is null. /// * bit_length is in number of bits pub fn bit_length(array: &dyn Array) -> Result { @@ -203,6 +190,15 @@ pub fn bit_length(array: &dyn Array) -> Result { array.nulls().cloned(), )?)) } + DataType::RunEndEncoded(k, _) => match k.data_type() { + DataType::Int16 => ree_map!(array, Int16Type, bit_length), + DataType::Int32 => ree_map!(array, Int32Type, bit_length), + DataType::Int64 => ree_map!(array, Int64Type, bit_length), + _ => Err(ArrowError::InvalidArgumentError(format!( + "Invalid run-end type: {:?}", + k.data_type() + ))), + }, other => Err(ArrowError::ComputeError(format!( "bit_length not supported for {other:?}" ))), @@ -903,4 +899,27 @@ mod tests { assert!(length(&ree_array).is_err()); } + + #[test] + fn bit_length_test_ree_utf8() { + use arrow_array::RunArray; + use arrow_array::types::Int32Type; + + let strings = StringArray::from(vec!["hello", "world", "test"]); + let run_ends = PrimitiveArray::::from(vec![1i32, 2, 3]); + let ree_array = RunArray::::try_new(&run_ends, &strings).unwrap(); + + let result = bit_length(&ree_array).unwrap(); + let result_values = result + .as_any() + .downcast_ref::>() + .unwrap() + .values() + .as_any() + .downcast_ref::() + .unwrap(); + + let expected: Int32Array = vec![40, 40, 32].into(); + assert_eq!(&expected, result_values); + } } diff --git a/arrow-string/src/like.rs b/arrow-string/src/like.rs index 55fe292ecb76..e9077d4b090d 100644 --- a/arrow-string/src/like.rs +++ b/arrow-string/src/like.rs @@ -15,7 +15,11 @@ // specific language governing permissions and limitations // under the License. -//! Provide SQL's LIKE operators for Arrow's string arrays +//! String predicate kernels for Arrow arrays. +//! +//! Provides SQL `LIKE`/`ILIKE` kernels as well as related +//! string predicates such as `contains`, `starts_with`, `ends_with`, and +//! ASCII case-insensitive equality. use crate::predicate::Predicate; @@ -24,8 +28,6 @@ use arrow_array::*; use arrow_schema::*; use arrow_select::take::take; -use std::sync::Arc; - use crate::binary_like::binary_apply; pub use arrow_array::StringArrayType; @@ -34,6 +36,7 @@ pub(crate) enum Op { Like(bool), ILike(bool), Contains, + EqIgnoreAsciiCase, StartsWith, EndsWith, } @@ -46,6 +49,7 @@ impl std::fmt::Display for Op { Op::ILike(false) => write!(f, "ILIKE"), Op::ILike(true) => write!(f, "NILIKE"), Op::Contains => write!(f, "CONTAINS"), + Op::EqIgnoreAsciiCase => write!(f, "EQ_IGNORE_ASCII_CASE"), Op::StartsWith => write!(f, "STARTS_WITH"), Op::EndsWith => write!(f, "ENDS_WITH"), } @@ -124,7 +128,7 @@ pub fn nilike(left: &dyn Datum, right: &dyn Datum) -> Result Result Result Result Result { + like_op(Op::EqIgnoreAsciiCase, left, right) +} + fn like_op(op: Op, lhs: &dyn Datum, rhs: &dyn Datum) -> Result { use arrow_schema::DataType::*; let (l, l_s) = lhs.get(); @@ -328,6 +356,7 @@ fn op_scalar<'a, T: StringArrayType<'a>>( Op::Like(neg) => Predicate::like(r)?.evaluate_array(l, neg), Op::ILike(neg) => Predicate::ilike(r, l.is_ascii())?.evaluate_array(l, neg), Op::Contains => Predicate::contains(r).evaluate_array(l, false), + Op::EqIgnoreAsciiCase => Predicate::IEqAscii(r).evaluate_array(l, false), Op::StartsWith => Predicate::StartsWith(r).evaluate_array(l, false), Op::EndsWith => Predicate::EndsWith(r).evaluate_array(l, false), }; @@ -362,6 +391,10 @@ fn op_binary<'a>( Op::Like(neg) => binary_predicate(l, r, neg, Predicate::like), Op::ILike(neg) => binary_predicate(l, r, neg, |s| Predicate::ilike(s, false)), Op::Contains => Ok(l.zip(r).map(|(l, r)| Some(str_contains(l?, r?))).collect()), + Op::EqIgnoreAsciiCase => Ok(l + .zip(r) + .map(|(l, r)| Some(Predicate::IEqAscii(l?).evaluate(r?))) + .collect()), Op::StartsWith => Ok(l .zip(r) .map(|(l, r)| Some(Predicate::StartsWith(r?).evaluate(l?))) @@ -398,118 +431,7 @@ fn binary_predicate<'a>( .collect() } -// Deprecated kernels - -fn make_scalar(data_type: &DataType, scalar: &str) -> Result { - match data_type { - DataType::Utf8 => Ok(Arc::new(StringArray::from_iter_values([scalar]))), - DataType::LargeUtf8 => Ok(Arc::new(LargeStringArray::from_iter_values([scalar]))), - DataType::Dictionary(_, v) => make_scalar(v.as_ref(), scalar), - d => Err(ArrowError::InvalidArgumentError(format!( - "Unsupported string scalar data type {d:?}", - ))), - } -} - -macro_rules! legacy_kernels { - ($fn_datum:ident, $fn_array:ident, $fn_scalar:ident, $fn_array_dyn:ident, $fn_scalar_dyn:ident, $deprecation:expr) => { - #[doc(hidden)] - #[deprecated(note = $deprecation)] - pub fn $fn_array( - left: &GenericStringArray, - right: &GenericStringArray, - ) -> Result { - $fn_datum(left, right) - } - - #[doc(hidden)] - #[deprecated(note = $deprecation)] - pub fn $fn_scalar( - left: &GenericStringArray, - right: &str, - ) -> Result { - let scalar = GenericStringArray::::from_iter_values([right]); - $fn_datum(left, &Scalar::new(&scalar)) - } - - #[doc(hidden)] - #[deprecated(note = $deprecation)] - pub fn $fn_array_dyn( - left: &dyn Array, - right: &dyn Array, - ) -> Result { - $fn_datum(&left, &right) - } - - #[doc(hidden)] - #[deprecated(note = $deprecation)] - pub fn $fn_scalar_dyn(left: &dyn Array, right: &str) -> Result { - let scalar = make_scalar(left.data_type(), right)?; - $fn_datum(&left, &Scalar::new(&scalar)) - } - }; -} - -legacy_kernels!( - like, - like_utf8, - like_utf8_scalar, - like_dyn, - like_utf8_scalar_dyn, - "Use arrow_string::like::like" -); -legacy_kernels!( - ilike, - ilike_utf8, - ilike_utf8_scalar, - ilike_dyn, - ilike_utf8_scalar_dyn, - "Use arrow_string::like::ilike" -); -legacy_kernels!( - nlike, - nlike_utf8, - nlike_utf8_scalar, - nlike_dyn, - nlike_utf8_scalar_dyn, - "Use arrow_string::like::nlike" -); -legacy_kernels!( - nilike, - nilike_utf8, - nilike_utf8_scalar, - nilike_dyn, - nilike_utf8_scalar_dyn, - "Use arrow_string::like::nilike" -); -legacy_kernels!( - contains, - contains_utf8, - contains_utf8_scalar, - contains_dyn, - contains_utf8_scalar_dyn, - "Use arrow_string::like::contains" -); -legacy_kernels!( - starts_with, - starts_with_utf8, - starts_with_utf8_scalar, - starts_with_dyn, - starts_with_utf8_scalar_dyn, - "Use arrow_string::like::starts_with" -); - -legacy_kernels!( - ends_with, - ends_with_utf8, - ends_with_utf8_scalar, - ends_with_dyn, - ends_with_utf8_scalar_dyn, - "Use arrow_string::like::ends_with" -); - #[cfg(test)] -#[allow(deprecated)] mod tests { use super::*; use arrow_array::builder::BinaryDictionaryBuilder; @@ -653,6 +575,10 @@ mod tests { assert_eq!(res, expected); let left: DictionaryArray = $left.into_iter().collect(); + let right = StringArray::from_iter_values([$right]); + let res = $op(&left, &Scalar::new(&right)).unwrap(); + assert_eq!(res, expected); + let right: DictionaryArray = [$right].into_iter().collect(); let res = $op(&left, &Scalar::new(&right)).unwrap(); assert_eq!(res, expected); @@ -1394,585 +1320,102 @@ mod tests { vec![true, false, true, true, true] ); - #[test] - fn test_dict_like_kernels() { - let data = vec![ + // Nullable, repeated values exercise dictionary remapping with a plain UTF8 scalar RHS. + test_utf8_scalar!( + test_utf8_scalar_nullable_like, + vec![ Some("Earth"), Some("Fire"), Some("Water"), Some("Air"), None, Some("Air"), - Some("bbbbb\nAir"), - ]; - - let dict_array: DictionaryArray = data.into_iter().collect(); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "Air").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(false), - Some(true), - None, - Some(true), - Some(false), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "Air").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(false), - Some(true), - None, - Some(true), - Some(false), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "Wa%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "Wa%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "%r").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "%r").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "%i%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "%i%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "%a%r%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "%a%r%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - } + Some("bbbbb\nAir") + ], + "Air", + like, + vec![ + Some(false), + Some(false), + Some(false), + Some(true), + None, + Some(true), + Some(false) + ] + ); - #[test] - fn test_dict_nlike_kernels() { - let data = vec![ + test_utf8_scalar!( + test_utf8_scalar_nullable_nlike, + vec![ Some("Earth"), Some("Fire"), Some("Water"), Some("Air"), None, Some("Air"), - Some("bbbbb\nAir"), - ]; - - let dict_array: DictionaryArray = data.into_iter().collect(); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "Air").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(true), - Some(false), - None, - Some(false), - Some(true), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "Air").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(true), - Some(false), - None, - Some(false), - Some(true), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "Wa%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "Wa%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "%r").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "%r").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "%i%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "%i%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "%a%r%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "%a%r%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - } + Some("bbbbb\nAir") + ], + "%a%r%", + nlike, + vec![ + Some(false), + Some(true), + Some(false), + Some(true), + None, + Some(true), + Some(true) + ] + ); - #[test] - fn test_dict_ilike_kernels() { - let data = vec![ + test_utf8_scalar!( + test_utf8_scalar_nullable_ilike, + vec![ Some("Earth"), Some("Fire"), Some("Water"), Some("Air"), None, Some("Air"), - Some("bbbbb\nAir"), - ]; - - let dict_array: DictionaryArray = data.into_iter().collect(); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "air").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(false), - Some(true), - None, - Some(true), - Some(false), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "air").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(false), - Some(true), - None, - Some(true), - Some(false), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "wa%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "wa%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "%R").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "%R").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "%I%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "%I%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "%A%r%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "%A%r%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - } + Some("bbbbb\nAir") + ], + "%I%", + ilike, + vec![ + Some(false), + Some(true), + Some(false), + Some(true), + None, + Some(true), + Some(true) + ] + ); - #[test] - fn test_dict_nilike_kernels() { - let data = vec![ + test_utf8_scalar!( + test_utf8_scalar_nullable_nilike, + vec![ Some("Earth"), Some("Fire"), Some("Water"), Some("Air"), None, Some("Air"), - Some("bbbbb\nAir"), - ]; - - let dict_array: DictionaryArray = data.into_iter().collect(); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "air").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(true), - Some(false), - None, - Some(false), - Some(true), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "air").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(true), - Some(false), - None, - Some(false), - Some(true), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "wa%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "wa%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "%R").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "%R").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "%I%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "%I%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "%A%r%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "%A%r%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - } + Some("bbbbb\nAir") + ], + "%R", + nilike, + vec![ + Some(true), + Some(true), + Some(false), + Some(false), + None, + Some(false), + Some(false) + ] + ); #[test] fn string_null_like_pattern() { diff --git a/arrow-string/src/substring.rs b/arrow-string/src/substring.rs index 05b3888a444a..3dc3bd695c44 100644 --- a/arrow-string/src/substring.rs +++ b/arrow-string/src/substring.rs @@ -86,8 +86,7 @@ pub fn substring( stringify!($gt), array.data_type()) }); let values = substring(dict.values(), start, length)?; - let result = DictionaryArray::try_new(dict.keys().clone(), values)?; - Ok(Arc::new(result)) + Ok(Arc::new(dict.with_values(values))) }, )* t => panic!("Unsupported dictionary key type: {}", t) diff --git a/arrow/benches/cast_kernels.rs b/arrow/benches/cast_kernels.rs index 2af3145a84e1..82e9696f0458 100644 --- a/arrow/benches/cast_kernels.rs +++ b/arrow/benches/cast_kernels.rs @@ -83,36 +83,6 @@ fn build_utf8_date_time_array(size: usize, with_nulls: bool) -> ArrayRef { Arc::new(builder.finish()) } -fn build_decimal32_array(size: usize, precision: u8, scale: i8) -> ArrayRef { - let mut rng = seedable_rng(); - let mut builder = Decimal32Builder::with_capacity(size); - - for _ in 0..size { - builder.append_value(rng.random_range::(0..1000000)); - } - Arc::new( - builder - .finish() - .with_precision_and_scale(precision, scale) - .unwrap(), - ) -} - -fn build_decimal64_array(size: usize, precision: u8, scale: i8) -> ArrayRef { - let mut rng = seedable_rng(); - let mut builder = Decimal64Builder::with_capacity(size); - - for _ in 0..size { - builder.append_value(rng.random_range::(0..1000000000)); - } - Arc::new( - builder - .finish() - .with_precision_and_scale(precision, scale) - .unwrap(), - ) -} - fn build_decimal128_array(size: usize, precision: u8, scale: i8) -> ArrayRef { let mut rng = seedable_rng(); let mut builder = Decimal128Builder::with_capacity(size); @@ -175,6 +145,14 @@ fn build_string_float_array(size: usize, null_density: f32) -> ArrayRef { Arc::new(builder.finish()) } +fn build_float64_array_for_cast_to_decimal(size: usize, null_density: f32) -> ArrayRef { + Arc::new(create_primitive_array_range::( + size, + null_density, + -999_999_999f64..999_999_999f64, + )) +} + macro_rules! build_array_with_samples { ($builder: ident, $size: ident, $null_density: expr, $samples: ident) => {{ let mut rng = seedable_rng(); @@ -189,46 +167,6 @@ macro_rules! build_array_with_samples { }}; } -fn build_string_float_array_invalid_item(size: usize, null_density: f32) -> ArrayRef { - let invalid_items = [ - "--1.23", - "1.2.3", - "-1.-23499", - "--1.23456789", - "1-.234", - "e10", - "1e", - "1e++10", - "NaN", - "Infinity", - ]; - let mut builder = StringBuilder::new(); - build_array_with_samples!(builder, size, null_density, invalid_items) -} - -fn build_float32_array_for_cast_to_decimal(size: usize, null_density: f32) -> ArrayRef { - Arc::new(create_primitive_array_range::( - size, - null_density, - -999_999_999f32..999_999_999f32, - )) -} - -fn build_float64_array_for_cast_to_decimal(size: usize, null_density: f32) -> ArrayRef { - Arc::new(create_primitive_array_range::( - size, - null_density, - -999_999_999f64..999_999_999f64, - )) -} - -fn build_float32_array_invalid_item(size: usize, null_density: f32) -> ArrayRef { - let mut builder = Float32Builder::with_capacity(size); - let invalid_values = [f32::NAN, f32::INFINITY, f32::NEG_INFINITY]; - - build_array_with_samples!(builder, size, null_density, invalid_values) -} - fn build_float64_array_invalid_items(size: usize, null_density: f32) -> ArrayRef { let mut builder = Float64Builder::with_capacity(size); let invalid_values = [f64::NAN, f64::INFINITY, f64::NEG_INFINITY]; @@ -268,8 +206,6 @@ fn add_benchmark(c: &mut Criterion) { let utf8_date_array = build_utf8_date_array(512, true); let utf8_date_time_array = build_utf8_date_time_array(512, true); - let decimal32_array = build_decimal32_array(8_000, 9, 3); - let decimal64_array = build_decimal64_array(8_000, 10, 3); let decimal128_array = build_decimal128_array(8_000, 10, 3); let decimal256_array = build_decimal256_array(8_000, 50, 3); let string_array = build_string_array(512); @@ -280,10 +216,7 @@ fn add_benchmark(c: &mut Criterion) { let binary_view_array = cast(&string_view_array, &DataType::BinaryView).unwrap(); let string_float_array_normal = build_string_float_array(5_000, 0.1); - let invalid_string_float_array = build_string_float_array_invalid_item(8_000, 0.1); - let float32_array_cast_to_decimal = build_float32_array_for_cast_to_decimal(8_000, 0.1); let float64_array_cast_to_decimal = build_float64_array_for_cast_to_decimal(8_000, 0.1); - let invalid_float32_array_to_decimal = build_float32_array_invalid_item(8_000, 0.1); let invalid_float64_array_to_decimal = build_float64_array_invalid_items(8_000, 0.1); c.bench_function("cast int32 to int32 512", |b| { @@ -366,22 +299,6 @@ fn add_benchmark(c: &mut Criterion) { b.iter(|| cast_array(&utf8_date_time_array, DataType::Date64)) }); - c.bench_function("cast decimal32 to decimal32 512", |b| { - b.iter(|| cast_array(&decimal32_array, DataType::Decimal32(9, 4))) - }); - c.bench_function("cast decimal32 to decimal32 512 lower precision", |b| { - b.iter(|| cast_array(&decimal32_array, DataType::Decimal32(6, 5))) - }); - c.bench_function("cast decimal32 to decimal64 512", |b| { - b.iter(|| cast_array(&decimal32_array, DataType::Decimal64(11, 5))) - }); - c.bench_function("cast decimal64 to decimal32 512", |b| { - b.iter(|| cast_array(&decimal64_array, DataType::Decimal32(9, 2))) - }); - c.bench_function("cast decimal64 to decimal64 512", |b| { - b.iter(|| cast_array(&decimal64_array, DataType::Decimal64(12, 4))) - }); - c.bench_function("cast decimal128 to decimal128 512", |b| { b.iter(|| cast_array(&decimal128_array, DataType::Decimal128(30, 5))) }); @@ -455,288 +372,44 @@ fn add_benchmark(c: &mut Criterion) { } // cast string with normal items to decimals - benchmark_cast!( - "cast string to decimal32(9, 2)", - string_float_array_normal, - DataType::Decimal32(9, 2) - ); - benchmark_cast!( - "cast string to decimal64(18, 2)", - string_float_array_normal, - DataType::Decimal64(18, 2) - ); benchmark_cast!( "cast string to decimal128(38, 3)", string_float_array_normal, DataType::Decimal128(38, 3) ); - benchmark_cast!( - "cast string to decimal256(76, 4)", - string_float_array_normal, - DataType::Decimal256(76, 4) - ); - - // cast invalid string to decimals - benchmark_cast!( - "cast invalid string to decimal32(9, 2)", - invalid_string_float_array, - DataType::Decimal32(9, 2) - ); - benchmark_cast!( - "cast invalid string to decimal64(18, 2)", - invalid_string_float_array, - DataType::Decimal64(18, 2) - ); - benchmark_cast!( - "cast invalid string to decimal128(38, 3)", - invalid_string_float_array, - DataType::Decimal128(38, 3) - ); - benchmark_cast!( - "cast invalid string to decimal256(76, 4)", - invalid_string_float_array, - DataType::Decimal256(76, 4) - ); - - // cast float32 to decimals - benchmark_cast!( - "cast float32 to decimal32(9, 2)", - float32_array_cast_to_decimal, - DataType::Decimal32(9, 2) - ); - benchmark_cast!( - "cast float32 to decimal64(18, 2", - float32_array_cast_to_decimal, - DataType::Decimal64(18, 2) - ); - benchmark_cast!( - "cast float32 to decimal128(32, 3)", - float32_array_cast_to_decimal, - DataType::Decimal128(38, 3) - ); - benchmark_cast!( - "cast float32 to decimal256(76, 4)", - float32_array_cast_to_decimal, - DataType::Decimal256(76, 4) - ); - - // cast invalid float32 to decimals - benchmark_cast!( - "cast invalid float32 to decimal32(9, 2)", - invalid_float32_array_to_decimal, - DataType::Decimal32(9, 2) - ); - benchmark_cast!( - "cast invalid float32 to decimal64(18, 2", - invalid_float32_array_to_decimal, - DataType::Decimal64(18, 2) - ); - benchmark_cast!( - "cast invalid float32 to decimal128(32, 3)", - invalid_float32_array_to_decimal, - DataType::Decimal128(32, 3) - ); - benchmark_cast!( - "cast invalid float32 to decimal256(76, 4)", - invalid_float32_array_to_decimal, - DataType::Decimal256(76, 4) - ); // cast float64 to decimals - benchmark_cast!( - "cast float64 to decimal32(9, 2)", - float64_array_cast_to_decimal, - DataType::Decimal32(9, 2) - ); - benchmark_cast!( - "cast float64 to decimal64(18, 2", - float64_array_cast_to_decimal, - DataType::Decimal64(18, 2) - ); benchmark_cast!( "cast float64 to decimal128(32, 3)", float64_array_cast_to_decimal, DataType::Decimal128(32, 3) ); - benchmark_cast!( - "cast float64 to decimal256(76, 4)", - float64_array_cast_to_decimal, - DataType::Decimal256(76, 4) - ); // cast invalid float64 to decimals - benchmark_cast!( - "cast invalid float64 to decimal32(9, 2)", - invalid_float64_array_to_decimal, - DataType::Decimal32(9, 2) - ); - benchmark_cast!( - "cast invalid float64 to to decimal64(18, 2)", - invalid_float64_array_to_decimal, - DataType::Decimal64(18, 2) - ); benchmark_cast!( "cast invalid float64 to to decimal128(32, 3)", invalid_float64_array_to_decimal, DataType::Decimal128(32, 3) ); - benchmark_cast!( - "cast invalid float64 to to decimal256(76, 4)", - invalid_float64_array_to_decimal, - DataType::Decimal256(76, 4) - ); // cast decimals to float/integers - benchmark_cast!( - "cast decimal32 to float32", - decimal32_array, - DataType::Float32 - ); - benchmark_cast!( - "cast decimal32 to float64", - decimal32_array, - DataType::Float64 - ); - benchmark_cast!("cast decimal32 to uint8", decimal32_array, DataType::UInt8); - benchmark_cast!( - "cast decimal32 to uint16", - decimal32_array, - DataType::UInt16 - ); - benchmark_cast!( - "cast decimal32 to uint32", - decimal32_array, - DataType::UInt32 - ); - benchmark_cast!( - "cast decimal32 to uint64", - decimal32_array, - DataType::UInt64 - ); - benchmark_cast!("cast decimal32 to int8", decimal32_array, DataType::Int8); - benchmark_cast!("cast decimal32 to int16", decimal32_array, DataType::Int16); - benchmark_cast!("cast decimal32 to int32", decimal32_array, DataType::Int32); - benchmark_cast!("cast decimal32 to int64", decimal32_array, DataType::Int64); - - benchmark_cast!( - "cast decimal64 to float32", - decimal64_array, - DataType::Float32 - ); - benchmark_cast!( - "cast decimal64 to float64", - decimal64_array, - DataType::Float64 - ); - benchmark_cast!("cast decimal64 to uint8", decimal64_array, DataType::UInt8); - benchmark_cast!( - "cast decimal64 to uint16", - decimal64_array, - DataType::UInt16 - ); - benchmark_cast!( - "cast decimal64 to uint32", - decimal64_array, - DataType::UInt32 - ); - benchmark_cast!( - "cast decimal64 to uint64", - decimal64_array, - DataType::UInt64 - ); - benchmark_cast!("cast decimal64 to int8", decimal64_array, DataType::Int8); - benchmark_cast!("cast decimal64 to int16", decimal64_array, DataType::Int16); - benchmark_cast!("cast decimal64 to int32", decimal64_array, DataType::Int32); - benchmark_cast!("cast decimal64 to int64", decimal64_array, DataType::Int64); - - benchmark_cast!( - "cast decimal128 to float32", - decimal128_array, - DataType::Float32 - ); benchmark_cast!( "cast decimal128 to float64", decimal128_array, DataType::Float64 ); - benchmark_cast!( - "cast decimal128 to uint8", - decimal128_array, - DataType::UInt8 - ); - benchmark_cast!( - "cast decimal128 to uint16", - decimal128_array, - DataType::UInt16 - ); - benchmark_cast!( - "cast decimal128 to uint32", - decimal128_array, - DataType::UInt32 - ); - benchmark_cast!( - "cast decimal128 to uint64", - decimal128_array, - DataType::UInt64 - ); benchmark_cast!("cast decimal128 to int8", decimal128_array, DataType::Int8); - benchmark_cast!( - "cast decimal128 to int16", - decimal128_array, - DataType::Int16 - ); - benchmark_cast!( - "cast decimal128 to int32", - decimal128_array, - DataType::Int32 - ); benchmark_cast!( "cast decimal128 to int64", decimal128_array, DataType::Int64 ); - benchmark_cast!( - "cast decimal256 to float32", - decimal256_array, - DataType::Float32 - ); benchmark_cast!( "cast decimal256 to float64", decimal256_array, DataType::Float64 ); - benchmark_cast!( - "cast decimal256 to uint8", - decimal256_array, - DataType::UInt8 - ); - benchmark_cast!( - "cast decimal256 to uint16", - decimal256_array, - DataType::UInt16 - ); - benchmark_cast!( - "cast decimal256 to uint32", - decimal256_array, - DataType::UInt32 - ); - benchmark_cast!( - "cast decimal256 to uint64", - decimal256_array, - DataType::UInt64 - ); - benchmark_cast!("cast decimal256 to int8", decimal256_array, DataType::Int8); - benchmark_cast!( - "cast decimal256 to int16", - decimal256_array, - DataType::Int16 - ); - benchmark_cast!( - "cast decimal256 to int32", - decimal256_array, - DataType::Int32 - ); benchmark_cast!( "cast decimal256 to int64", decimal256_array, diff --git a/arrow/benches/interleave_kernels.rs b/arrow/benches/interleave_kernels.rs index 0688bb79adb1..4c4f5364ffe6 100644 --- a/arrow/benches/interleave_kernels.rs +++ b/arrow/benches/interleave_kernels.rs @@ -150,6 +150,14 @@ fn add_benchmark(c: &mut Criterion) { ) }; + let ree_run_ends = Int32Array::from_iter_values((1..=64).map(|i| i * 16)); + let ree_i64_values = create_primitive_array::(64, 0.0); + let ree_i64 = RunArray::::try_new(&ree_run_ends, &ree_i64_values).unwrap(); + + let dict_str_values = create_string_array_with_len::(20, 0.0, 12); + let dict_for_ree = create_dict_from_values::(64, 0.0, &dict_str_values); + let ree_dict = RunArray::::try_new(&ree_run_ends, &dict_for_ree).unwrap(); + let cases: &[(&str, &dyn Array)] = &[ ("i32(0.0)", &i32), ("i32(0.5)", &i32_opt), @@ -175,6 +183,8 @@ fn add_benchmark(c: &mut Criterion) { ("list_view(0.1,0.1,20)", &list_view_i64), ("list_view(0.0,0.0,20)", &list_view_i64_no_nulls), ("list_view_overlapping(80x,20)", &list_view_overlapping), + ("ree_i32(64 runs)", &ree_i64), + ("ree_i32>(64 runs)", &ree_dict), ]; for (prefix, base) in cases { diff --git a/dev/release/update_change_log.sh b/dev/release/update_change_log.sh index d4c8fa250141..21b4c2d2956d 100755 --- a/dev/release/update_change_log.sh +++ b/dev/release/update_change_log.sh @@ -29,8 +29,8 @@ set -e -SINCE_TAG="58.1.0" -FUTURE_RELEASE="58.2.0" +SINCE_TAG="58.2.0" +FUTURE_RELEASE="58.3.0" SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" diff --git a/parquet/Cargo.toml b/parquet/Cargo.toml index 9f4d2a33df0c..2edeec40635e 100644 --- a/parquet/Cargo.toml +++ b/parquet/Cargo.toml @@ -289,5 +289,10 @@ required-features = ["arrow"] name = "bloom_filter" harness = false +[[bench]] +name = "arrow_reader_peak_memory" +required-features = ["arrow", "test_common", "experimental"] +harness = false + [lib] bench = false diff --git a/parquet/benches/arrow_reader.rs b/parquet/benches/arrow_reader.rs index 9325a1faf5b6..888ff5083ad0 100644 --- a/parquet/benches/arrow_reader.rs +++ b/parquet/benches/arrow_reader.rs @@ -95,6 +95,16 @@ fn build_test_schema() -> SchemaDescPtr { OPTIONAL GROUP optional_struct_optional_int32_leaf { OPTIONAL INT32 element; } + OPTIONAL GROUP int32_list (LIST) { + repeated group list { + optional INT32 element; + } + } + OPTIONAL GROUP fixed32_list (LIST) { + repeated group list { + optional FIXED_LEN_BYTE_ARRAY(32) element; + } + } } "; parse_message_type(message_type) @@ -668,6 +678,150 @@ fn build_string_list_page_iterator( InMemoryPageIterator::new(pages) } +fn build_int32_list_page_iterator( + column_desc: ColumnDescPtr, + null_density: f32, +) -> impl PageIterator + Clone { + let max_def_level = column_desc.max_def_level(); + let max_rep_level = column_desc.max_rep_level(); + assert_eq!(max_def_level, 3); + assert_eq!(max_rep_level, 1); + + let mut rng = seedable_rng(); + let mut pages: Vec> = Vec::new(); + for _i in 0..NUM_ROW_GROUPS { + let mut column_chunk_pages = Vec::new(); + for _j in 0..PAGES_PER_GROUP { + let mut values: Vec = Vec::with_capacity(VALUES_PER_PAGE * MAX_LIST_LEN); + let mut def_levels = Vec::with_capacity(VALUES_PER_PAGE * MAX_LIST_LEN); + let mut rep_levels = Vec::with_capacity(VALUES_PER_PAGE * MAX_LIST_LEN); + for _k in 0..VALUES_PER_PAGE { + rep_levels.push(0); + if rng.random::() < null_density { + def_levels.push(0); + continue; + } + let len = rng.random_range(0..MAX_LIST_LEN); + if len == 0 { + def_levels.push(1); + continue; + } + + (1..len).for_each(|_| rep_levels.push(1)); + + for _l in 0..len { + if rng.random::() < null_density { + def_levels.push(2); + } else { + def_levels.push(3); + values.push(rng.random()); + } + } + } + let mut page_builder = + DataPageBuilderImpl::new(column_desc.clone(), values.len() as u32, true); + page_builder.add_rep_levels(max_rep_level, &rep_levels); + page_builder.add_def_levels(max_def_level, &def_levels); + page_builder.add_values::(Encoding::PLAIN, &values); + column_chunk_pages.push(page_builder.consume()); + } + pages.push(column_chunk_pages); + } + + InMemoryPageIterator::new(pages) +} + +fn create_int32_list_reader( + page_iterator: impl PageIterator + 'static, + column_desc: ColumnDescPtr, +) -> Box { + use parquet::arrow::array_reader::PrimitiveArrayReader; + let items = Box::new( + PrimitiveArrayReader::::new( + Box::new(page_iterator), + column_desc, + None, + DEFAULT_BATCH_SIZE, + ) + .unwrap(), + ) as Box; + let field = Field::new_list_field(DataType::Int32, true); + let data_type = DataType::List(Arc::new(field)); + Box::new(ListArrayReader::::new(items, data_type, 2, 1, true)) +} + +const FIXED_BYTE_LEN: usize = 32; + +fn build_fixed32_list_page_iterator( + column_desc: ColumnDescPtr, + null_density: f32, +) -> impl PageIterator + Clone { + let max_def_level = column_desc.max_def_level(); + let max_rep_level = column_desc.max_rep_level(); + assert_eq!(max_def_level, 3); + assert_eq!(max_rep_level, 1); + + let mut rng = seedable_rng(); + let mut pages: Vec> = Vec::new(); + for _i in 0..NUM_ROW_GROUPS { + let mut column_chunk_pages = Vec::new(); + for _j in 0..PAGES_PER_GROUP { + let mut values: Vec = + Vec::with_capacity(VALUES_PER_PAGE * MAX_LIST_LEN); + let mut def_levels = Vec::with_capacity(VALUES_PER_PAGE * MAX_LIST_LEN); + let mut rep_levels = Vec::with_capacity(VALUES_PER_PAGE * MAX_LIST_LEN); + for _k in 0..VALUES_PER_PAGE { + rep_levels.push(0); + if rng.random::() < null_density { + def_levels.push(0); + continue; + } + let len = rng.random_range(0..MAX_LIST_LEN); + if len == 0 { + def_levels.push(1); + continue; + } + (1..len).for_each(|_| rep_levels.push(1)); + for _l in 0..len { + if rng.random::() < null_density { + def_levels.push(2); + } else { + def_levels.push(3); + let mut buf = vec![0u8; FIXED_BYTE_LEN]; + rng.fill(&mut buf[..]); + values.push(buf.into()); + } + } + } + let mut page_builder = + DataPageBuilderImpl::new(column_desc.clone(), values.len() as u32, true); + page_builder.add_rep_levels(max_rep_level, &rep_levels); + page_builder.add_def_levels(max_def_level, &def_levels); + page_builder.add_values::(Encoding::PLAIN, &values); + column_chunk_pages.push(page_builder.consume()); + } + pages.push(column_chunk_pages); + } + + InMemoryPageIterator::new(pages) +} + +fn create_fixed32_list_reader( + page_iterator: impl PageIterator + 'static, + column_desc: ColumnDescPtr, +) -> Box { + let items = make_fixed_len_byte_array_reader( + Box::new(page_iterator), + column_desc, + None, + DEFAULT_BATCH_SIZE, + ) + .unwrap(); + let field = Field::new_list_field(DataType::FixedSizeBinary(FIXED_BYTE_LEN as i32), true); + let data_type = DataType::List(Arc::new(field)); + Box::new(ListArrayReader::::new(items, data_type, 2, 1, true)) +} + fn bench_array_reader(mut array_reader: Box) -> usize { // test procedure: read data in batches of 8192 until no more data let mut total_count = 0; @@ -1645,6 +1799,8 @@ fn add_benches(c: &mut Criterion) { let optional_uint64_column_desc = schema.column(38); let mandatory_struct_optional_in32_column_desc = schema.column(39); let optional_struct_optional_in32_column_desc = schema.column(40); + let int32_list_desc = schema.column(41); + let fixed32_list_desc = schema.column(42); // primitive / int32 benchmarks // ============================= @@ -2228,24 +2384,59 @@ fn add_benches(c: &mut Criterion) { // list benchmarks //============================== - let list_data = build_string_list_page_iterator(string_list_desc.clone(), 0.); - let mut group = c.benchmark_group("arrow_array_reader/ListArray"); - group.bench_function("plain encoded optional strings no NULLs", |b| { - b.iter(|| { - let reader = create_string_list_reader(list_data.clone(), string_list_desc.clone()); - count = bench_array_reader(reader); + let mut group = c.benchmark_group("arrow_array_reader/ListArray/StringList"); + for (label, null_density) in [ + ("no NULLs", 0.0), + ("half NULLs", 0.5), + ("90pct NULLs", 0.9), + ("99pct NULLs", 0.99), + ] { + let list_data = build_string_list_page_iterator(string_list_desc.clone(), null_density); + group.bench_function(label, |b| { + b.iter(|| { + let reader = create_string_list_reader(list_data.clone(), string_list_desc.clone()); + count = bench_array_reader(reader); + }); + assert_eq!(count, EXPECTED_VALUE_COUNT); }); - assert_eq!(count, EXPECTED_VALUE_COUNT); - }); - let list_data = build_string_list_page_iterator(string_list_desc.clone(), 0.5); - group.bench_function("plain encoded optional strings half NULLs", |b| { - b.iter(|| { - let reader = create_string_list_reader(list_data.clone(), string_list_desc.clone()); - count = bench_array_reader(reader); + } + group.finish(); + + let mut group = c.benchmark_group("arrow_array_reader/ListArray/Int32List"); + for (label, null_density) in [ + ("no NULLs", 0.0), + ("half NULLs", 0.5), + ("90pct NULLs", 0.9), + ("99pct NULLs", 0.99), + ] { + let list_data = build_int32_list_page_iterator(int32_list_desc.clone(), null_density); + group.bench_function(label, |b| { + b.iter(|| { + let reader = create_int32_list_reader(list_data.clone(), int32_list_desc.clone()); + count = bench_array_reader(reader); + }); + assert_eq!(count, EXPECTED_VALUE_COUNT); }); - assert_eq!(count, EXPECTED_VALUE_COUNT); - }); + } + group.finish(); + let mut group = c.benchmark_group("arrow_array_reader/ListArray/Fixed32List"); + for (label, null_density) in [ + ("no NULLs", 0.0), + ("half NULLs", 0.5), + ("90pct NULLs", 0.9), + ("99pct NULLs", 0.99), + ] { + let list_data = build_fixed32_list_page_iterator(fixed32_list_desc.clone(), null_density); + group.bench_function(label, |b| { + b.iter(|| { + let reader = + create_fixed32_list_reader(list_data.clone(), fixed32_list_desc.clone()); + count = bench_array_reader(reader); + }); + assert_eq!(count, EXPECTED_VALUE_COUNT); + }); + } group.finish(); // fixed_len_byte_array benchmarks diff --git a/parquet/benches/arrow_reader_peak_memory.rs b/parquet/benches/arrow_reader_peak_memory.rs new file mode 100644 index 000000000000..6486bf2da0bf --- /dev/null +++ b/parquet/benches/arrow_reader_peak_memory.rs @@ -0,0 +1,597 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Criterion benchmark measuring memory usage of ListArrayReader for sparse +//! list columns. +//! +//! Uses a global allocator with thread-local memory tracking so that +//! concurrent criterion threads don't interfere with measurements. + +use std::alloc::Layout; +use std::cell::Cell; +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field}; +use criterion::measurement::{Measurement, ValueFormatter}; +use criterion::{BenchmarkGroup, Criterion, Throughput, criterion_group, criterion_main}; +use parquet::arrow::array_reader::make_byte_array_reader; +use parquet::arrow::array_reader::{ + ArrayReader, ListArrayReader, PrimitiveArrayReader, make_fixed_len_byte_array_reader, +}; +use parquet::arrow::arrow_reader::DEFAULT_BATCH_SIZE; +use parquet::basic::Encoding; +use parquet::column::page::PageIterator; +use parquet::data_type::{ByteArrayType, DoubleType, FixedLenByteArrayType, Int32Type}; +use parquet::schema::parser::parse_message_type; +use parquet::schema::types::{ColumnDescPtr, SchemaDescriptor}; +use parquet::util::{DataPageBuilder, DataPageBuilderImpl, InMemoryPageIterator}; +use rand::{Rng, SeedableRng, rngs::StdRng}; + +// --------------------------------------------------------------------------- +// Thread-local tracking allocator +// --------------------------------------------------------------------------- + +thread_local! { + static LIVE_BYTES: Cell = const { Cell::new(0) }; + static PEAK_BYTES: Cell = const { Cell::new(0) }; + static ALLOCATED_BYTES: Cell = const { Cell::new(0) }; +} + +struct TrackingAllocator { + inner: std::alloc::System, +} + +#[global_allocator] +static GLOBAL: TrackingAllocator = TrackingAllocator { + inner: std::alloc::System, +}; + +fn add_live_bytes(size: usize) { + LIVE_BYTES.with(|live| { + let new = live.get().saturating_add(size); + live.set(new); + PEAK_BYTES.with(|peak| { + if new > peak.get() { + peak.set(new); + } + }); + }); +} + +fn subtract_live_bytes(size: usize) { + LIVE_BYTES.with(|live| { + live.set(live.get().saturating_sub(size)); + }); +} + +fn add_allocated_bytes(size: usize) { + ALLOCATED_BYTES.with(|allocated| { + allocated.set(allocated.get().saturating_add(size)); + }); +} + +#[allow(unsafe_code)] +unsafe impl std::alloc::GlobalAlloc for TrackingAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let ptr = unsafe { self.inner.alloc(layout) }; + if !ptr.is_null() { + add_live_bytes(layout.size()); + add_allocated_bytes(layout.size()); + } + ptr + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + subtract_live_bytes(layout.size()); + unsafe { self.inner.dealloc(ptr, layout) }; + } + + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + let new_ptr = unsafe { self.inner.realloc(ptr, layout, new_size) }; + if !new_ptr.is_null() { + let old_size = layout.size(); + add_allocated_bytes(new_size); + if new_size > old_size { + add_live_bytes(new_size - old_size); + } else { + subtract_live_bytes(old_size - new_size); + } + } + new_ptr + } +} + +fn reset_peak() { + PEAK_BYTES.with(|peak| { + LIVE_BYTES.with(|live| { + peak.set(live.get()); + }); + }); +} + +fn peak_bytes() -> usize { + PEAK_BYTES.with(|peak| peak.get()) +} + +fn live_bytes() -> usize { + LIVE_BYTES.with(|live| live.get()) +} + +fn reset_allocated() { + ALLOCATED_BYTES.with(|allocated| allocated.set(0)); +} + +fn allocated_bytes() -> usize { + ALLOCATED_BYTES.with(|allocated| allocated.get()) +} + +// --------------------------------------------------------------------------- +// Criterion custom measurements +// --------------------------------------------------------------------------- + +struct BytesFormatter; + +const BYTE_UNITS: &[(u32, &str)] = &[ + (60, "EiB"), + (50, "PiB"), + (40, "TiB"), + (30, "GiB"), + (20, "MiB"), + (10, "KiB"), + (0, "B"), +]; + +fn bytes_per_unit(exponent: u32) -> f64 { + (1_u64 << exponent) as f64 +} + +fn scale_bytes(typical: f64, values: &mut [f64]) -> &'static str { + for &(exponent, unit) in BYTE_UNITS { + let scale = bytes_per_unit(exponent); + if typical >= scale { + for v in values.iter_mut() { + *v /= scale; + } + return unit; + } + } + unreachable!("BYTE_UNITS contains B") +} + +impl ValueFormatter for BytesFormatter { + fn scale_values(&self, typical: f64, values: &mut [f64]) -> &'static str { + scale_bytes(typical, values) + } + + fn scale_throughputs( + &self, + typical: f64, + _throughput: &Throughput, + values: &mut [f64], + ) -> &'static str { + scale_bytes(typical, values) + } + + fn scale_for_machines(&self, values: &mut [f64]) -> &'static str { + // Machine-readable: always bytes + let _ = values; + "B" + } +} + +struct PeakMemory; + +impl Measurement for PeakMemory { + type Intermediate = usize; + type Value = usize; + + fn start(&self) -> Self::Intermediate { + reset_peak(); + live_bytes() + } + + fn end(&self, baseline: Self::Intermediate) -> Self::Value { + peak_bytes().saturating_sub(baseline) + } + + fn add(&self, v1: &Self::Value, v2: &Self::Value) -> Self::Value { + *v1 + *v2 + } + + fn zero(&self) -> Self::Value { + 0 + } + + fn to_f64(&self, value: &Self::Value) -> f64 { + *value as f64 + } + + fn formatter(&self) -> &dyn ValueFormatter { + &BytesFormatter + } +} + +struct AllocatedBytes; + +impl Measurement for AllocatedBytes { + type Intermediate = (); + type Value = usize; + + fn start(&self) -> Self::Intermediate { + reset_allocated(); + } + + fn end(&self, _baseline: Self::Intermediate) -> Self::Value { + allocated_bytes() + } + + fn add(&self, v1: &Self::Value, v2: &Self::Value) -> Self::Value { + *v1 + *v2 + } + + fn zero(&self) -> Self::Value { + 0 + } + + fn to_f64(&self, value: &Self::Value) -> f64 { + *value as f64 + } + + fn formatter(&self) -> &dyn ValueFormatter { + &BytesFormatter + } +} + +// --------------------------------------------------------------------------- +// Test data generation +// --------------------------------------------------------------------------- + +const NUM_ROW_GROUPS: usize = 2; +const PAGES_PER_GROUP: usize = 4; +const VALUES_PER_PAGE: usize = 10_000; +const MAX_LIST_LEN: usize = 10; +const BATCH_SIZE: usize = 8192; +const EXPECTED_VALUE_COUNT: usize = NUM_ROW_GROUPS * PAGES_PER_GROUP * VALUES_PER_PAGE; + +fn seedable_rng() -> StdRng { + StdRng::seed_from_u64(42) +} + +fn build_int32_list_schema() -> (SchemaDescriptor, ColumnDescPtr) { + let message_type = " + message schema { + OPTIONAL GROUP int32_list (LIST) { + repeated group list { + optional INT32 element; + } + } + } + "; + let schema = parse_message_type(message_type) + .map(|t| SchemaDescriptor::new(Arc::new(t))) + .unwrap(); + let col_desc = schema.column(0); + (schema, col_desc) +} + +fn build_string_list_schema() -> (SchemaDescriptor, ColumnDescPtr) { + let message_type = " + message schema { + OPTIONAL GROUP string_list (LIST) { + repeated group list { + optional BYTE_ARRAY element (UTF8); + } + } + } + "; + let schema = parse_message_type(message_type) + .map(|t| SchemaDescriptor::new(Arc::new(t))) + .unwrap(); + let col_desc = schema.column(0); + (schema, col_desc) +} + +const FIXED_BYTE_LEN: usize = 32; + +fn build_fixed32_list_schema() -> (SchemaDescriptor, ColumnDescPtr) { + let message_type = " + message schema { + OPTIONAL GROUP fixed32_list (LIST) { + repeated group list { + optional FIXED_LEN_BYTE_ARRAY(32) element; + } + } + } + "; + let schema = parse_message_type(message_type) + .map(|t| SchemaDescriptor::new(Arc::new(t))) + .unwrap(); + let col_desc = schema.column(0); + (schema, col_desc) +} + +fn build_double_list_schema() -> (SchemaDescriptor, ColumnDescPtr) { + let message_type = " + message schema { + OPTIONAL GROUP double_list (LIST) { + repeated group list { + optional DOUBLE element; + } + } + } + "; + let schema = parse_message_type(message_type) + .map(|t| SchemaDescriptor::new(Arc::new(t))) + .unwrap(); + let col_desc = schema.column(0); + (schema, col_desc) +} + +fn build_list_pages( + column_desc: ColumnDescPtr, + null_density: f32, + mut gen_value: impl FnMut(&mut StdRng) -> T::T, +) -> impl PageIterator + Clone { + let max_def_level = column_desc.max_def_level(); + let max_rep_level = column_desc.max_rep_level(); + let mut rng = seedable_rng(); + let mut pages: Vec> = Vec::new(); + + for _ in 0..NUM_ROW_GROUPS { + let mut column_chunk_pages = Vec::new(); + for _ in 0..PAGES_PER_GROUP { + let mut values = Vec::new(); + let mut def_levels = Vec::new(); + let mut rep_levels = Vec::new(); + + for _ in 0..VALUES_PER_PAGE { + rep_levels.push(0); + if rng.random::() < null_density { + def_levels.push(0); + continue; + } + let len = rng.random_range(0..MAX_LIST_LEN); + if len == 0 { + def_levels.push(1); + continue; + } + (1..len).for_each(|_| rep_levels.push(1)); + for _ in 0..len { + if rng.random::() < null_density { + def_levels.push(2); + } else { + def_levels.push(3); + values.push(gen_value(&mut rng)); + } + } + } + + let mut pb = DataPageBuilderImpl::new(column_desc.clone(), values.len() as u32, true); + pb.add_rep_levels(max_rep_level, &rep_levels); + pb.add_def_levels(max_def_level, &def_levels); + pb.add_values::(Encoding::PLAIN, &values); + column_chunk_pages.push(pb.consume()); + } + pages.push(column_chunk_pages); + } + InMemoryPageIterator::new(pages) +} + +fn drain_reader(mut reader: Box) -> usize { + let mut total_count = 0; + loop { + let batch = reader.next_batch(BATCH_SIZE).unwrap(); + let batch_len = batch.len(); + total_count += batch_len; + if batch_len < BATCH_SIZE { + break; + } + } + total_count +} + +// --------------------------------------------------------------------------- +// Benchmarks +// --------------------------------------------------------------------------- + +fn null_density_label(null_density: f32) -> &'static str { + match (null_density * 100.0) as u32 { + 0 => "no NULLs", + 50 => "half NULLs", + 90 => "90pct NULLs", + 99 => "99pct NULLs", + _ => unreachable!(), + } +} + +fn bench_list_memory( + group: &mut BenchmarkGroup, + null_density: f32, + pages: impl PageIterator + Clone + 'static, + make_reader: fn(Box, ColumnDescPtr) -> Box, + column_desc: &ColumnDescPtr, +) { + group.bench_function(null_density_label(null_density), |b| { + b.iter_batched( + || make_reader(Box::new(pages.clone()), column_desc.clone()), + |reader| { + let count = drain_reader(reader); + assert_eq!(count, EXPECTED_VALUE_COUNT); + count + }, + criterion::BatchSize::PerIteration, + ); + }); +} + +fn int32_list_wrapper( + pages: Box, + column_desc: ColumnDescPtr, +) -> Box { + let child: Box = Box::new( + PrimitiveArrayReader::::new(pages, column_desc, None, DEFAULT_BATCH_SIZE) + .unwrap(), + ); + let field = Field::new_list_field(DataType::Int32, true); + let data_type = DataType::List(Arc::new(field)); + Box::new(ListArrayReader::::new(child, data_type, 2, 1, true)) +} + +fn double_list_wrapper( + pages: Box, + column_desc: ColumnDescPtr, +) -> Box { + let child: Box = Box::new( + PrimitiveArrayReader::::new(pages, column_desc, None, DEFAULT_BATCH_SIZE) + .unwrap(), + ); + let field = Field::new_list_field(DataType::Float64, true); + let data_type = DataType::List(Arc::new(field)); + Box::new(ListArrayReader::::new(child, data_type, 2, 1, true)) +} + +fn fixed32_list_wrapper( + pages: Box, + column_desc: ColumnDescPtr, +) -> Box { + let child = + make_fixed_len_byte_array_reader(pages, column_desc, None, DEFAULT_BATCH_SIZE).unwrap(); + let field = Field::new_list_field(DataType::FixedSizeBinary(FIXED_BYTE_LEN as i32), true); + let data_type = DataType::List(Arc::new(field)); + Box::new(ListArrayReader::::new(child, data_type, 2, 1, true)) +} + +fn string_list_wrapper( + pages: Box, + column_desc: ColumnDescPtr, +) -> Box { + let child = make_byte_array_reader(pages, column_desc, None, DEFAULT_BATCH_SIZE).unwrap(); + let field = Field::new_list_field(DataType::Utf8, true); + let data_type = DataType::List(Arc::new(field)); + Box::new(ListArrayReader::::new(child, data_type, 2, 1, true)) +} + +fn add_benches(c: &mut Criterion, measurement_name: &str) { + let (_schema, int32_desc) = build_int32_list_schema(); + let (_schema, double_desc) = build_double_list_schema(); + let (_schema, fixed32_desc) = build_fixed32_list_schema(); + let (_schema, string_desc) = build_string_list_schema(); + + let mut group = c.benchmark_group(format!( + "arrow_array_reader/ListArray_{measurement_name}/Int32List" + )); + for null_density in [0.0, 0.5, 0.9, 0.99] { + let pages = + build_list_pages::(int32_desc.clone(), null_density, |rng| rng.random()); + bench_list_memory( + &mut group, + null_density, + pages, + int32_list_wrapper, + &int32_desc, + ); + } + group.finish(); + + let mut group = c.benchmark_group(format!( + "arrow_array_reader/ListArray_{measurement_name}/DoubleList" + )); + for null_density in [0.0, 0.5, 0.9, 0.99] { + let pages = + build_list_pages::(double_desc.clone(), null_density, |rng| rng.random()); + bench_list_memory( + &mut group, + null_density, + pages, + double_list_wrapper, + &double_desc, + ); + } + group.finish(); + + let mut group = c.benchmark_group(format!( + "arrow_array_reader/ListArray_{measurement_name}/Fixed32List" + )); + for null_density in [0.0, 0.5, 0.9, 0.99] { + let pages = + build_list_pages::(fixed32_desc.clone(), null_density, |rng| { + let mut buf = vec![0u8; FIXED_BYTE_LEN]; + rng.fill(&mut buf[..]); + buf.into() + }); + bench_list_memory( + &mut group, + null_density, + pages, + fixed32_list_wrapper, + &fixed32_desc, + ); + } + group.finish(); + + let mut group = c.benchmark_group(format!( + "arrow_array_reader/ListArray_{measurement_name}/StringList" + )); + for null_density in [0.0, 0.5, 0.9, 0.99] { + let pages = build_list_pages::(string_desc.clone(), null_density, |rng| { + let len = rng.random_range(5..50); + let bytes: Vec = (0..len).map(|_| rng.random_range(b'a'..=b'z')).collect(); + bytes.into() + }); + bench_list_memory( + &mut group, + null_density, + pages, + string_list_wrapper, + &string_desc, + ); + } + group.finish(); +} + +fn add_peak_memory_benches(c: &mut Criterion) { + add_benches(c, "peak_memory"); +} + +fn add_allocated_bytes_benches(c: &mut Criterion) { + add_benches(c, "allocated_bytes"); +} + +fn peak_memory_criterion() -> Criterion { + Criterion::default() + .with_measurement(PeakMemory) + .sample_size(10) +} + +fn allocated_bytes_criterion() -> Criterion { + Criterion::default() + .with_measurement(AllocatedBytes) + .sample_size(10) +} + +criterion_group! { + name = peak_memory; + config = peak_memory_criterion(); + targets = add_peak_memory_benches +} + +criterion_group! { + name = cumulative_allocated_bytes; + config = allocated_bytes_criterion(); + targets = add_allocated_bytes_benches +} +criterion_main!(peak_memory, cumulative_allocated_bytes); diff --git a/parquet/benches/metadata.rs b/parquet/benches/metadata.rs index c9a6cf3b762c..b6e04580c141 100644 --- a/parquet/benches/metadata.rs +++ b/parquet/benches/metadata.rs @@ -41,7 +41,7 @@ use parquet::file::serialized_reader::ReadOptionsBuilder; const NUM_COLUMNS: usize = 10_000; const NUM_ROW_GROUPS: usize = 10; -fn encoded_meta(is_nullable: bool, has_lists: bool) -> Vec { +fn encoded_meta(is_nullable: bool, has_lists: bool, write_path_in_schema: bool) -> Vec { let mut rng = seedable_rng(); let mut column_desc_ptrs: Vec = Vec::with_capacity(NUM_COLUMNS); @@ -90,7 +90,7 @@ fn encoded_meta(is_nullable: bool, has_lists: bool) -> Vec { .map(|j| { ColumnChunkMetaData::builder(column_desc_ptrs[j].clone()) .set_encodings(vec![Encoding::PLAIN, Encoding::RLE_DICTIONARY]) - .set_compression(parquet::basic::Compression::UNCOMPRESSED) + .set_compression_codec(parquet::basic::CompressionCodec::UNCOMPRESSED) .set_num_values(rng.random_range(1..1000000)) .set_total_compressed_size(rng.random_range(50000..5000000)) .set_data_page_offset(rng.random_range(4..2000000000)) @@ -143,7 +143,11 @@ fn encoded_meta(is_nullable: bool, has_lists: bool) -> Vec { let mut buffer = Vec::with_capacity(1024); { let buf = TrackedWrite::new(&mut buffer); - let writer = ParquetMetaDataWriter::new_with_tracked(buf, &metadata); + let mut writer = ParquetMetaDataWriter::new_with_tracked(buf, &metadata); + // use defaults unless `write_path_in_schema` is false + if !write_path_in_schema { + writer = writer.with_write_path_in_schema(write_path_in_schema); + } writer.finish().unwrap(); } @@ -233,7 +237,7 @@ fn criterion_benchmark(c: &mut Criterion) { }) }); - let buf: Bytes = black_box(encoded_meta(false, false)).into(); + let buf: Bytes = black_box(encoded_meta(false, false, true)).into(); let options = ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false); c.bench_function("decode parquet metadata (wide)", |b| { b.iter(|| { @@ -275,7 +279,15 @@ fn criterion_benchmark(c: &mut Criterion) { }) }); - let buf: Bytes = black_box(encoded_meta(true, true)).into(); + let buf: Bytes = black_box(encoded_meta(false, false, false)).into(); + let options = ParquetMetaDataOptions::new().with_encoding_stats_as_mask(false); + c.bench_function("decode parquet metadata no path_in_schema (wide)", |b| { + b.iter(|| { + ParquetMetaDataReader::decode_metadata_with_options(&buf, Some(&options)).unwrap(); + }) + }); + + let buf: Bytes = black_box(encoded_meta(true, true, true)).into(); c.bench_function("decode parquet metadata w/ size stats (wide)", |b| { b.iter(|| { ParquetMetaDataReader::decode_metadata(&buf).unwrap(); diff --git a/parquet/src/arrow/array_reader/fixed_size_list_array.rs b/parquet/src/arrow/array_reader/fixed_size_list_array.rs index 8ef3bd6c2a4b..518cd8625ed5 100644 --- a/parquet/src/arrow/array_reader/fixed_size_list_array.rs +++ b/parquet/src/arrow/array_reader/fixed_size_list_array.rs @@ -226,7 +226,7 @@ mod tests { use super::*; use crate::arrow::{ ArrowWriter, - array_reader::{ListArrayReader, test_util::InMemoryArrayReader}, + array_reader::{ListArrayReader, test_util::make_int32_page_reader}, arrow_reader::{ArrowReaderBuilder, ArrowReaderOptions, ParquetRecordBatchReader}, }; use arrow::datatypes::{Field, Int32Type}; @@ -254,28 +254,16 @@ mod tests { 3, ); - let array = Arc::new(PrimitiveArray::::from(vec![ - None, - Some(1), - None, - Some(2), - None, - Some(3), - Some(4), - Some(5), - None, - None, - None, - ])); - let item_array_reader = InMemoryArrayReader::new( - ArrowType::Int32, - array, - Some(vec![0, 3, 2, 3, 0, 3, 3, 3, 2, 2, 2]), - Some(vec![0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1]), + let item_array_reader = make_int32_page_reader( + &[1, 2, 3, 4, 5], + &[0, 3, 2, 3, 0, 3, 3, 3, 2, 2, 2], + &[0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1], + 3, + 1, ); let mut list_array_reader = FixedSizeListArrayReader::new( - Box::new(item_array_reader), + item_array_reader, 3, ArrowType::FixedSizeList(Arc::new(Field::new_list_field(ArrowType::Int32, true)), 3), 2, @@ -303,25 +291,16 @@ mod tests { 2, ); - let array = Arc::new(PrimitiveArray::::from(vec![ - Some(1), - None, - Some(2), - Some(3), - None, - None, - Some(4), - Some(5), - ])); - let item_array_reader = InMemoryArrayReader::new( - ArrowType::Int32, - array, - Some(vec![2, 1, 2, 2, 1, 1, 2, 2]), - Some(vec![0, 1, 0, 1, 0, 1, 0, 1]), + let item_array_reader = make_int32_page_reader( + &[1, 2, 3, 4, 5], + &[2, 1, 2, 2, 1, 1, 2, 2], + &[0, 1, 0, 1, 0, 1, 0, 1], + 2, + 1, ); let mut list_array_reader = FixedSizeListArrayReader::new( - Box::new(item_array_reader), + item_array_reader, 2, ArrowType::FixedSizeList(Arc::new(Field::new_list_field(ArrowType::Int32, true)), 2), 1, @@ -381,28 +360,15 @@ mod tests { let expected = FixedSizeListArray::from(l1); - let values = Arc::new(PrimitiveArray::::from(vec![ - None, - Some(1), - Some(2), - None, - Some(3), - None, - Some(4), - Some(5), - None, - None, - ])); - - let item_array_reader = InMemoryArrayReader::new( - ArrowType::Int32, - values, - Some(vec![0, 5, 5, 4, 5, 0, 5, 5, 4, 4]), - Some(vec![0, 0, 2, 0, 2, 0, 0, 2, 0, 2]), + let item_array_reader = make_int32_page_reader( + &[1, 2, 3, 4, 5], + &[0, 5, 5, 4, 5, 0, 5, 5, 4, 4], + &[0, 0, 2, 0, 2, 0, 0, 2, 0, 2], + 5, + 2, ); - let l2 = - FixedSizeListArrayReader::new(Box::new(item_array_reader), 2, l2_type, 4, 2, false); + let l2 = FixedSizeListArrayReader::new(item_array_reader, 2, l2_type, 4, 2, false); let mut l1 = FixedSizeListArrayReader::new(Box::new(l2), 1, l1_type, 3, 1, true); let expected_1 = expected.slice(0, 2); @@ -423,18 +389,10 @@ mod tests { 0, ); - let array = Arc::new(PrimitiveArray::::from(vec![ - None, None, None, None, - ])); - let item_array_reader = InMemoryArrayReader::new( - ArrowType::Int32, - array, - Some(vec![0, 1, 0, 1]), - Some(vec![0, 0, 0, 0]), - ); + let item_array_reader = make_int32_page_reader(&[], &[0, 1, 0, 1], &[0, 0, 0, 0], 2, 1); let mut list_array_reader = FixedSizeListArrayReader::new( - Box::new(item_array_reader), + item_array_reader, 0, ArrowType::FixedSizeList(Arc::new(Field::new_list_field(ArrowType::Int32, true)), 0), 2, @@ -467,33 +425,20 @@ mod tests { builder.append(false); let expected = builder.finish(); - let array = Arc::new(PrimitiveArray::::from(vec![ - Some(1), - None, - Some(3), - None, - Some(4), - None, - Some(5), - Some(6), - None, - None, - None, - ])); - let inner_type = ArrowType::List(Arc::new(Field::new_list_field(ArrowType::Int32, true))); let list_type = ArrowType::FixedSizeList(Arc::new(Field::new_list_field(inner_type.clone(), true)), 2); - let item_array_reader = InMemoryArrayReader::new( - ArrowType::Int32, - array, - Some(vec![5, 4, 5, 2, 5, 3, 5, 5, 4, 4, 0]), - Some(vec![0, 2, 2, 1, 0, 1, 0, 2, 1, 2, 0]), + let item_array_reader = make_int32_page_reader( + &[1, 3, 4, 5, 6], + &[5, 4, 5, 2, 5, 3, 5, 5, 4, 4, 0], + &[0, 2, 2, 1, 0, 1, 0, 2, 1, 2, 0], + 5, + 2, ); let inner_array_reader = - ListArrayReader::::new(Box::new(item_array_reader), inner_type, 4, 2, true); + ListArrayReader::::new(item_array_reader, inner_type, 4, 2, true); let mut list_array_reader = FixedSizeListArrayReader::new(Box::new(inner_array_reader), 2, list_type, 2, 1, true); diff --git a/parquet/src/arrow/array_reader/list_array.rs b/parquet/src/arrow/array_reader/list_array.rs index a7adc01b912d..e6c834096902 100644 --- a/parquet/src/arrow/array_reader/list_array.rs +++ b/parquet/src/arrow/array_reader/list_array.rs @@ -248,7 +248,7 @@ mod tests { use super::*; use crate::arrow::array_reader::ArrayReaderBuilder; use crate::arrow::array_reader::list_array::ListArrayReader; - use crate::arrow::array_reader::test_util::InMemoryArrayReader; + use crate::arrow::array_reader::test_util::make_int32_page_reader; use crate::arrow::arrow_reader::DEFAULT_BATCH_SIZE; use crate::arrow::arrow_reader::metrics::ArrowReaderMetrics; use crate::arrow::schema::parquet_to_arrow_schema_and_fields; @@ -257,7 +257,7 @@ mod tests { use crate::file::reader::{FileReader, SerializedFileReader}; use crate::schema::parser::parse_message_type; use crate::schema::types::SchemaDescriptor; - use arrow::datatypes::{Field, Int32Type as ArrowInt32, Int32Type}; + use arrow::datatypes::{Field, Int32Type}; use arrow_array::{Array, PrimitiveArray}; use arrow_data::ArrayDataBuilder; use arrow_schema::Fields; @@ -354,36 +354,15 @@ mod tests { let expected = GenericListArray::::from(l1); - let values = Arc::new(PrimitiveArray::::from(vec![ - Some(1), - None, - None, - Some(4), - None, - None, - Some(7), - None, - Some(1), - Some(2), - Some(3), - Some(4), - None, - Some(6), - None, - None, - None, - Some(11), - ])); - - let item_array_reader = InMemoryArrayReader::new( - ArrowType::Int32, - values, - Some(vec![6, 5, 3, 6, 4, 2, 6, 4, 6, 6, 6, 6, 5, 6, 3, 0, 1, 6]), - Some(vec![0, 3, 2, 2, 2, 1, 1, 1, 1, 3, 3, 2, 3, 3, 2, 0, 0, 0]), + let item_array_reader = make_int32_page_reader( + &[1, 4, 7, 1, 2, 3, 4, 6, 11], + &[6, 5, 3, 6, 4, 2, 6, 4, 6, 6, 6, 6, 5, 6, 3, 0, 1, 6], + &[0, 3, 2, 2, 2, 1, 1, 1, 1, 3, 3, 2, 3, 3, 2, 0, 0, 0], + 6, + 3, ); - let l3 = - ListArrayReader::::new(Box::new(item_array_reader), l3_type, 5, 3, true); + let l3 = ListArrayReader::::new(item_array_reader, l3_type, 5, 3, true); let l2 = ListArrayReader::::new(Box::new(l3), l2_type, 3, 2, false); @@ -411,28 +390,16 @@ mod tests { Some(vec![None, Some(1)]), ]); - let array = Arc::new(PrimitiveArray::::from(vec![ - Some(1), - None, - Some(2), - None, - Some(3), - Some(4), - None, - None, - None, - Some(1), - ])); - - let item_array_reader = InMemoryArrayReader::new( - ArrowType::Int32, - array, - Some(vec![2, 1, 2, 0, 2, 2, 0, 0, 1, 2]), - Some(vec![0, 1, 1, 0, 0, 1, 0, 0, 0, 1]), + let item_array_reader = make_int32_page_reader( + &[1, 2, 3, 4, 1], + &[2, 1, 2, 0, 2, 2, 0, 0, 1, 2], + &[0, 1, 1, 0, 0, 1, 0, 0, 0, 1], + 2, + 1, ); let mut list_array_reader = ListArrayReader::::new( - Box::new(item_array_reader), + item_array_reader, list_type::(ArrowType::Int32, true), 1, 1, @@ -460,31 +427,16 @@ mod tests { Some(vec![None, Some(1)]), ]); - let array = Arc::new(PrimitiveArray::::from(vec![ - Some(1), - None, - Some(2), - None, - None, - Some(3), - Some(4), - None, - None, - None, - None, - None, - Some(1), - ])); - - let item_array_reader = InMemoryArrayReader::new( - ArrowType::Int32, - array, - Some(vec![3, 2, 3, 0, 1, 3, 3, 1, 1, 0, 1, 2, 3]), - Some(vec![0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1]), + let item_array_reader = make_int32_page_reader( + &[1, 2, 3, 4, 1], + &[3, 2, 3, 0, 1, 3, 3, 1, 1, 0, 1, 2, 3], + &[0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1], + 3, + 1, ); let mut list_array_reader = ListArrayReader::::new( - Box::new(item_array_reader), + item_array_reader, list_type::(ArrowType::Int32, true), 2, 1, diff --git a/parquet/src/arrow/array_reader/list_view_array.rs b/parquet/src/arrow/array_reader/list_view_array.rs index 294135d41f8b..357ab9dc14ae 100644 --- a/parquet/src/arrow/array_reader/list_view_array.rs +++ b/parquet/src/arrow/array_reader/list_view_array.rs @@ -106,9 +106,8 @@ impl ArrayReader for ListViewArrayReader() { // [[1, null, 2], null, [], [3, 4], [], [], null, [], [null, 1]] @@ -125,27 +124,12 @@ mod tests { Some(vec![None, Some(1)]), ]); - let array = Arc::new(PrimitiveArray::::from(vec![ - Some(1), - None, - Some(2), - None, - None, - Some(3), - Some(4), - None, - None, - None, - None, - None, - Some(1), - ])); - - let item_array_reader = InMemoryArrayReader::new( - ArrowType::Int32, - array, - Some(vec![3, 2, 3, 0, 1, 3, 3, 1, 1, 0, 1, 2, 3]), - Some(vec![0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1]), + let item_array_reader = make_int32_page_reader( + &[1, 2, 3, 4, 1], + &[3, 2, 3, 0, 1, 3, 3, 1, 1, 0, 1, 2, 3], + &[0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1], + 3, + 1, ); let field = Arc::new(arrow_schema::Field::new_list_field(ArrowType::Int32, true)); @@ -155,13 +139,8 @@ mod tests { ArrowType::ListView(field) }; - let mut list_view_array_reader = ListViewArrayReader::::new( - Box::new(item_array_reader), - data_type, - 2, - 1, - true, - ); + let mut list_view_array_reader = + ListViewArrayReader::::new(item_array_reader, data_type, 2, 1, true); let actual = list_view_array_reader.next_batch(1024).unwrap(); let actual = actual @@ -184,24 +163,12 @@ mod tests { Some(vec![None, Some(1)]), ]); - let array = Arc::new(PrimitiveArray::::from(vec![ - Some(1), - None, - Some(2), - None, - Some(3), - Some(4), - None, - None, - None, - Some(1), - ])); - - let item_array_reader = InMemoryArrayReader::new( - ArrowType::Int32, - array, - Some(vec![2, 1, 2, 0, 2, 2, 0, 0, 1, 2]), - Some(vec![0, 1, 1, 0, 0, 1, 0, 0, 0, 1]), + let item_array_reader = make_int32_page_reader( + &[1, 2, 3, 4, 1], + &[2, 1, 2, 0, 2, 2, 0, 0, 1, 2], + &[0, 1, 1, 0, 0, 1, 0, 0, 0, 1], + 2, + 1, ); let field = Arc::new(arrow_schema::Field::new_list_field(ArrowType::Int32, true)); @@ -211,13 +178,8 @@ mod tests { ArrowType::ListView(field) }; - let mut list_view_array_reader = ListViewArrayReader::::new( - Box::new(item_array_reader), - data_type, - 1, - 1, - false, - ); + let mut list_view_array_reader = + ListViewArrayReader::::new(item_array_reader, data_type, 1, 1, false); let actual = list_view_array_reader.next_batch(1024).unwrap(); let actual = actual diff --git a/parquet/src/arrow/array_reader/struct_array.rs b/parquet/src/arrow/array_reader/struct_array.rs index 63b20eb0d655..da92d410f32b 100644 --- a/parquet/src/arrow/array_reader/struct_array.rs +++ b/parquet/src/arrow/array_reader/struct_array.rs @@ -224,39 +224,27 @@ impl ArrayReader for StructArrayReader { mod tests { use super::*; use crate::arrow::array_reader::ListArrayReader; - use crate::arrow::array_reader::test_util::InMemoryArrayReader; + use crate::arrow::array_reader::test_util::make_int32_page_reader; use arrow::buffer::Buffer; use arrow::datatypes::Field; use arrow_array::cast::AsArray; - use arrow_array::{Array, Int32Array, ListArray}; + use arrow_array::{Array, ListArray}; use arrow_schema::Fields; #[test] fn test_struct_array_reader() { - let array_1 = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5])); - let array_reader_1 = InMemoryArrayReader::new( - ArrowType::Int32, - array_1.clone(), - Some(vec![0, 1, 2, 3, 1]), - Some(vec![0, 1, 1, 1, 1]), - ); + let array_reader_1 = make_int32_page_reader(&[4], &[0, 1, 2, 3, 1], &[0, 1, 1, 1, 1], 3, 1); - let array_2 = Arc::new(Int32Array::from(vec![5, 4, 3, 2, 1])); - let array_reader_2 = InMemoryArrayReader::new( - ArrowType::Int32, - array_2.clone(), - Some(vec![0, 1, 3, 1, 2]), - Some(vec![0, 1, 1, 1, 1]), - ); + let array_reader_2 = make_int32_page_reader(&[3], &[0, 1, 3, 1, 2], &[0, 1, 1, 1, 1], 3, 1); let struct_type = ArrowType::Struct(Fields::from(vec![ - Field::new("f1", array_1.data_type().clone(), true), - Field::new("f2", array_2.data_type().clone(), true), + Field::new("f1", ArrowType::Int32, true), + Field::new("f2", ArrowType::Int32, true), ])); let mut struct_array_reader = StructArrayReader::new( struct_type, - vec![Box::new(array_reader_1), Box::new(array_reader_2)], + vec![array_reader_1, array_reader_2], 1, 1, true, @@ -306,28 +294,11 @@ mod tests { )]; let expected = StructArray::from((struct_fields, validity)); - let array = Arc::new(Int32Array::from_iter(vec![ - Some(1), - Some(2), - None, - None, - None, - None, - ])); - let reader = InMemoryArrayReader::new( - ArrowType::Int32, - array, - Some(vec![4, 4, 3, 2, 1, 0]), - Some(vec![0, 1, 1, 0, 0, 0]), - ); + let reader = + make_int32_page_reader(&[1, 2], &[4, 4, 3, 2, 1, 0], &[0, 1, 1, 0, 0, 0], 4, 1); - let list_reader = ListArrayReader::::new( - Box::new(reader), - expected_l.data_type().clone(), - 3, - 1, - true, - ); + let list_reader = + ListArrayReader::::new(reader, expected_l.data_type().clone(), 3, 1, true); let mut struct_reader = StructArrayReader::new( expected.data_type().clone(), diff --git a/parquet/src/arrow/array_reader/test_util.rs b/parquet/src/arrow/array_reader/test_util.rs index 45af5a2777b3..afba7c2c30c2 100644 --- a/parquet/src/arrow/array_reader/test_util.rs +++ b/parquet/src/arrow/array_reader/test_util.rs @@ -15,16 +15,13 @@ // specific language governing permissions and limitations // under the License. -use arrow_array::{Array, ArrayRef}; -use arrow_schema::DataType as ArrowType; use bytes::Bytes; -use std::any::Any; use std::sync::Arc; use crate::arrow::array_reader::ArrayReader; use crate::basic::{ConvertedType, Encoding, Type as PhysicalType}; use crate::column::page::{PageIterator, PageReader}; -use crate::data_type::{ByteArray, ByteArrayType}; +use crate::data_type::{ByteArray, ByteArrayType, Int32Type}; use crate::encodings::encoding::{DictEncoder, Encoder, get_encoder}; use crate::errors::Result; use crate::schema::types::{ColumnDescPtr, ColumnDescriptor, ColumnPath, Type}; @@ -91,112 +88,51 @@ pub fn byte_array_all_encodings( (pages, encoded_dictionary) } -/// Array reader for test. -pub struct InMemoryArrayReader { - data_type: ArrowType, - array: ArrayRef, - def_levels: Option>, - rep_levels: Option>, - last_idx: usize, - cur_idx: usize, - need_consume_records: usize, -} - -impl InMemoryArrayReader { - pub fn new( - data_type: ArrowType, - array: ArrayRef, - def_levels: Option>, - rep_levels: Option>, - ) -> Self { - assert!( - def_levels - .as_ref() - .map(|d| d.len() == array.len()) - .unwrap_or(true) - ); - - assert!( - rep_levels - .as_ref() - .map(|r| r.len() == array.len()) - .unwrap_or(true) - ); - - Self { - data_type, - array, - def_levels, - rep_levels, - cur_idx: 0, - last_idx: 0, - need_consume_records: 0, - } - } -} - -impl ArrayReader for InMemoryArrayReader { - fn as_any(&self) -> &dyn Any { - self - } - - fn get_data_type(&self) -> &ArrowType { - &self.data_type - } - - fn read_records(&mut self, batch_size: usize) -> Result { - assert_ne!(batch_size, 0); - // This replicates the logical normally performed by - // RecordReader to delimit semantic records - let read = match &self.rep_levels { - Some(rep_levels) => { - let rep_levels = &rep_levels[self.cur_idx..]; - let mut levels_read = 0; - let mut records_read = 0; - while levels_read < rep_levels.len() && records_read < batch_size { - if rep_levels[levels_read] == 0 { - records_read += 1; // Start of new record - } - levels_read += 1; - } - - // Find end of current record - while levels_read < rep_levels.len() && rep_levels[levels_read] != 0 { - levels_read += 1 - } - levels_read - } - None => batch_size.min(self.array.len() - self.cur_idx), - }; - self.need_consume_records += read; - Ok(read) - } - - fn consume_batch(&mut self) -> Result { - let batch_size = self.need_consume_records; - assert_ne!(batch_size, 0); - self.last_idx = self.cur_idx; - self.cur_idx += batch_size; - self.need_consume_records = 0; - Ok(self.array.slice(self.last_idx, batch_size)) - } +/// Build a real `PrimitiveArrayReader` from raw non-null values +/// and definition/repetition levels. This exercises the full production +/// `RecordReader` code path (including selective padding when a parent +/// `ListArrayReader` calls `enable_selective_padding`). +/// +/// `values` must contain only the non-null values (entries where +/// `def_levels[i] == max_def_level`), in order, as Parquet encodes them. +pub fn make_int32_page_reader( + values: &[i32], + def_levels: &[i16], + rep_levels: &[i16], + max_def_level: i16, + max_rep_level: i16, +) -> Box { + use crate::arrow::array_reader::PrimitiveArrayReader; + use crate::arrow::arrow_reader::DEFAULT_BATCH_SIZE; + use crate::util::InMemoryPageIterator; + use crate::util::test_common::page_util::{DataPageBuilder, DataPageBuilderImpl}; + + let leaf_type = Type::primitive_type_builder("leaf", PhysicalType::INT32) + .build() + .unwrap(); - fn skip_records(&mut self, num_records: usize) -> Result { - let array = self.next_batch(num_records)?; - Ok(array.len()) - } + let desc = Arc::new(ColumnDescriptor::new( + Arc::new(leaf_type), + max_def_level, + max_rep_level, + ColumnPath::new(vec![]), + )); - fn get_def_levels(&self) -> Option<&[i16]> { - self.def_levels - .as_ref() - .map(|l| &l[self.last_idx..self.cur_idx]) + let mut pb = DataPageBuilderImpl::new(desc.clone(), def_levels.len() as u32, true); + if max_rep_level > 0 { + pb.add_rep_levels(max_rep_level, rep_levels); } - - fn get_rep_levels(&self) -> Option<&[i16]> { - self.rep_levels - .as_ref() - .map(|l| &l[self.last_idx..self.cur_idx]) + if max_def_level > 0 { + pb.add_def_levels(max_def_level, def_levels); } + pb.add_values::(Encoding::PLAIN, values); + + let pages = vec![vec![pb.consume()]]; + let page_iter = InMemoryPageIterator::new(pages); + Box::new( + PrimitiveArrayReader::::new(Box::new(page_iter), desc, None, DEFAULT_BATCH_SIZE) + .unwrap(), + ) } /// Iterator for testing reading empty columns diff --git a/parquet/src/arrow/arrow_reader/read_plan.rs b/parquet/src/arrow/arrow_reader/read_plan.rs index ac2e105ecf4f..674ae2b8d964 100644 --- a/parquet/src/arrow/arrow_reader/read_plan.rs +++ b/parquet/src/arrow/arrow_reader/read_plan.rs @@ -536,11 +536,9 @@ mod tests { fn with_predicate_options_limit_pads_tail_when_no_prior_selection() { use crate::arrow::ProjectionMask; use crate::arrow::array_reader::StructArrayReader; - use crate::arrow::array_reader::test_util::InMemoryArrayReader; + use crate::arrow::array_reader::test_util::make_int32_page_reader; use crate::arrow::arrow_reader::ArrowPredicateFn; - use arrow_array::Int32Array; use arrow_schema::{DataType as ArrowType, Field, Fields}; - use std::sync::Arc; // 100 rows, all match the predicate. Limit stops the loop after 10 // matches — but the resulting RowSelection must still describe the @@ -550,14 +548,14 @@ mod tests { const LIMIT: usize = 10; let data: Vec = (0..TOTAL_ROWS as i32).collect(); - let array = Arc::new(Int32Array::from(data)); - let leaf = InMemoryArrayReader::new(ArrowType::Int32, array.clone(), None, None); + let levels = vec![0; TOTAL_ROWS]; + let leaf = make_int32_page_reader(&data, &levels, &levels, 0, 0); let struct_type = ArrowType::Struct(Fields::from(vec![Field::new( "c0", ArrowType::Int32, false, )])); - let struct_reader = StructArrayReader::new(struct_type, vec![Box::new(leaf)], 0, 0, false); + let struct_reader = StructArrayReader::new(struct_type, vec![leaf], 0, 0, false); let mut predicate = ArrowPredicateFn::new(ProjectionMask::all(), |batch| { Ok(BooleanArray::from(vec![true; batch.num_rows()])) diff --git a/parquet/src/arrow/arrow_writer/levels.rs b/parquet/src/arrow/arrow_writer/levels.rs index d23873278ed3..1f66de349df0 100644 --- a/parquet/src/arrow/arrow_writer/levels.rs +++ b/parquet/src/arrow/arrow_writer/levels.rs @@ -41,6 +41,7 @@ //! \[1\] [parquet-format#nested-encoding](https://github.com/apache/parquet-format#nested-encoding) use crate::column::chunker::CdcChunk; +use crate::column::writer::LevelDataRef; use crate::errors::{ParquetError, Result}; use arrow_array::cast::AsArray; use arrow_array::{Array, ArrayRef, OffsetSizeTrait}; @@ -316,7 +317,7 @@ impl LevelInfoBuilder { |child: &mut LevelInfoBuilder, start_idx: usize, end_idx: usize| { child.write(start_idx..end_idx); child.visit_leaves(|leaf| { - let rep_levels = leaf.rep_levels.as_mut().unwrap(); + let rep_levels = leaf.rep_levels.materialize_mut().unwrap(); let mut rev = rep_levels.iter_mut().rev(); let mut remaining = end_idx - start_idx; @@ -336,17 +337,18 @@ impl LevelInfoBuilder { }) }; + // In a list column, each row falls into one of three categories: + // - "null": the list slot is absent (!is_valid), encoded at def_level - 2 + // - "empty": the list slot is present but has zero elements + // (offsets[i] == offsets[i+1]), encoded at def_level - 1 + // - non-empty: the list slot has child values, which are recursed into + // + // Consecutive runs of null or empty rows are batched and written together. let write_null_run = |child: &mut LevelInfoBuilder, count: usize| { if count > 0 { child.visit_leaves(|leaf| { - leaf.rep_levels - .as_mut() - .unwrap() - .extend(std::iter::repeat_n(ctx.rep_level - 1, count)); - leaf.def_levels - .as_mut() - .unwrap() - .extend(std::iter::repeat_n(ctx.def_level - 2, count)); + leaf.append_rep_level_run(ctx.rep_level - 1, count); + leaf.append_def_level_run(ctx.def_level - 2, count); }); } }; @@ -354,14 +356,8 @@ impl LevelInfoBuilder { let write_empty_run = |child: &mut LevelInfoBuilder, count: usize| { if count > 0 { child.visit_leaves(|leaf| { - leaf.rep_levels - .as_mut() - .unwrap() - .extend(std::iter::repeat_n(ctx.rep_level - 1, count)); - leaf.def_levels - .as_mut() - .unwrap() - .extend(std::iter::repeat_n(ctx.def_level - 1, count)); + leaf.append_rep_level_run(ctx.rep_level - 1, count); + leaf.append_def_level_run(ctx.def_level - 1, count); }); } }; @@ -431,7 +427,7 @@ impl LevelInfoBuilder { |child: &mut LevelInfoBuilder, start_idx: usize, end_idx: usize| { child.write(start_idx..end_idx); child.visit_leaves(|leaf| { - let rep_levels = leaf.rep_levels.as_mut().unwrap(); + let rep_levels = leaf.rep_levels.materialize_mut().unwrap(); let mut rev = rep_levels.iter_mut().rev(); let mut remaining = end_idx - start_idx; @@ -453,19 +449,15 @@ impl LevelInfoBuilder { let write_empty_slice = |child: &mut LevelInfoBuilder| { child.visit_leaves(|leaf| { - let rep_levels = leaf.rep_levels.as_mut().unwrap(); - rep_levels.push(ctx.rep_level - 1); - let def_levels = leaf.def_levels.as_mut().unwrap(); - def_levels.push(ctx.def_level - 1); + leaf.append_rep_level_run(ctx.rep_level - 1, 1); + leaf.append_def_level_run(ctx.def_level - 1, 1); }) }; let write_null_slice = |child: &mut LevelInfoBuilder| { child.visit_leaves(|leaf| { - let rep_levels = leaf.rep_levels.as_mut().unwrap(); - rep_levels.push(ctx.rep_level - 1); - let def_levels = leaf.def_levels.as_mut().unwrap(); - def_levels.push(ctx.def_level - 2); + leaf.append_rep_level_run(ctx.rep_level - 1, 1); + leaf.append_def_level_run(ctx.def_level - 2, 1); }) }; @@ -513,13 +505,8 @@ impl LevelInfoBuilder { for child in children { child.visit_leaves(|info| { let len = range.end - range.start; - - let def_levels = info.def_levels.as_mut().unwrap(); - def_levels.extend(std::iter::repeat_n(ctx.def_level - 1, len)); - - if let Some(rep_levels) = info.rep_levels.as_mut() { - rep_levels.extend(std::iter::repeat_n(ctx.rep_level, len)); - } + info.append_def_level_run(ctx.def_level - 1, len); + info.append_rep_level_run(ctx.rep_level, len); }) } }; @@ -579,7 +566,7 @@ impl LevelInfoBuilder { child.write(values_start..values_end); child.visit_leaves(|leaf| { - let rep_levels = leaf.rep_levels.as_mut().unwrap(); + let rep_levels = leaf.rep_levels.materialize_mut().unwrap(); let row_indices = (0..fixed_size) .rev() @@ -605,10 +592,8 @@ impl LevelInfoBuilder { let write_empty = |child: &mut LevelInfoBuilder, start_idx: usize, end_idx: usize| { let len = end_idx - start_idx; child.visit_leaves(|leaf| { - let rep_levels = leaf.rep_levels.as_mut().unwrap(); - rep_levels.extend(std::iter::repeat_n(ctx.rep_level - 1, len)); - let def_levels = leaf.def_levels.as_mut().unwrap(); - def_levels.extend(std::iter::repeat_n(ctx.def_level - 1, len)); + leaf.append_rep_level_run(ctx.rep_level - 1, len); + leaf.append_def_level_run(ctx.def_level - 1, len); }) }; @@ -634,10 +619,8 @@ impl LevelInfoBuilder { } // Add null row child.visit_leaves(|leaf| { - let rep_levels = leaf.rep_levels.as_mut().unwrap(); - rep_levels.push(ctx.rep_level - 1); - let def_levels = leaf.def_levels.as_mut().unwrap(); - def_levels.push(ctx.def_level - 2); + leaf.append_rep_level_run(ctx.rep_level - 1, 1); + leaf.append_def_level_run(ctx.def_level - 2, 1); }) } } @@ -655,37 +638,34 @@ impl LevelInfoBuilder { fn write_leaf(info: &mut ArrayLevels, range: Range) { let len = range.end - range.start; - match &mut info.def_levels { - Some(def_levels) => { - def_levels.reserve(len); - info.non_null_indices.reserve(len); - - match &info.logical_nulls { - Some(nulls) => { - assert!(range.end <= nulls.len()); - let nulls = nulls.inner(); - def_levels.extend(range.clone().map(|i| { - // Safety: range.end was asserted to be in bounds earlier - let valid = unsafe { nulls.value_unchecked(i) }; - info.max_def_level - (!valid as i16) - })); - info.non_null_indices.extend( - BitIndexIterator::new(nulls.inner(), nulls.offset() + range.start, len) - .map(|i| i + range.start), - ); - } - None => { - let iter = std::iter::repeat_n(info.max_def_level, len); - def_levels.extend(iter); - info.non_null_indices.extend(range); - } + if matches!(info.def_levels, LevelData::Absent) { + info.non_null_indices.extend(range.clone()); + } else { + let max_def_level = info.max_def_level; + match &info.logical_nulls { + Some(nulls) => { + assert!(range.end <= nulls.len()); + let nulls = nulls.inner(); + info.def_levels.extend_from_iter(range.clone().map(|i| { + // Safety: range.end was asserted to be in bounds earlier + let valid = unsafe { nulls.value_unchecked(i) }; + max_def_level - (!valid as i16) + })); + info.non_null_indices.reserve(len); + info.non_null_indices.extend( + BitIndexIterator::new(nulls.inner(), nulls.offset() + range.start, len) + .map(|i| i + range.start), + ); + } + None => { + info.append_def_level_run(max_def_level, len); + info.non_null_indices.extend(range.clone()); } } - None => info.non_null_indices.extend(range), } - if let Some(rep_levels) = &mut info.rep_levels { - rep_levels.extend(std::iter::repeat_n(info.max_rep_level, len)) + if !matches!(info.rep_levels, LevelData::Absent) { + info.append_rep_level_run(info.max_rep_level, len); } } @@ -754,17 +734,142 @@ impl LevelInfoBuilder { /// The data necessary to write a primitive Arrow array to parquet, taking into account /// any non-primitive parents it may have in the arrow representation +#[derive(Debug, Clone)] +pub(crate) enum LevelData { + Absent, + Materialized(Vec), + Uniform { value: i16, count: usize }, +} + +// Compare logical level contents rather than physical representation, so a +// uniform run compares equal to the equivalent materialized buffer. +impl PartialEq for LevelData { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Absent, Self::Absent) => true, + (Self::Materialized(a), Self::Materialized(b)) => a == b, + (Self::Uniform { value: v, count: n }, Self::Materialized(b)) + | (Self::Materialized(b), Self::Uniform { value: v, count: n }) => { + b.len() == *n && b.iter().all(|x| x == v) + } + ( + Self::Uniform { + value: v1, + count: n1, + }, + Self::Uniform { + value: v2, + count: n2, + }, + ) => v1 == v2 && n1 == n2, + _ => false, + } + } +} + +impl Eq for LevelData {} + +impl LevelData { + fn new(present: bool) -> Self { + match present { + true => Self::Materialized(Vec::new()), + false => Self::Absent, + } + } + + pub(crate) fn as_ref(&self) -> LevelDataRef<'_> { + match self { + Self::Absent => LevelDataRef::Absent, + Self::Materialized(values) => LevelDataRef::Materialized(values), + Self::Uniform { value, count } => LevelDataRef::Uniform { + value: *value, + count: *count, + }, + } + } + + pub(crate) fn slice(&self, offset: usize, len: usize) -> Self { + match self { + Self::Absent => Self::Absent, + Self::Materialized(values) => Self::Materialized(values[offset..offset + len].to_vec()), + Self::Uniform { value, .. } => Self::Uniform { + value: *value, + count: len, + }, + } + } + + fn append_run(&mut self, value: i16, count: usize) { + if count == 0 { + return; + } + + match self { + // No physical level stream exists for this schema. Higher-level + // traversal may still append implicit levels, so this remains a no-op. + Self::Absent => {} + // Start compact: the first appended run can be represented without + // allocating a level buffer. + Self::Materialized(values) if values.is_empty() => { + *self = Self::Uniform { value, count }; + } + // Already materialized, so preserve the buffer representation and append. + Self::Materialized(values) => values.extend(std::iter::repeat_n(value, count)), + // Preserve the compact representation while the appended run has + // the same value. + Self::Uniform { + value: uniform_value, + count: uniform_count, + } if *uniform_value == value => { + *uniform_count += count; + } + // A different value breaks the uniform representation. Materialize + // the existing run, then append the new run to the buffer. + Self::Uniform { .. } => { + let values = self.materialize_mut().unwrap(); + values.extend(std::iter::repeat_n(value, count)); + } + } + } + + fn extend_from_iter(&mut self, iter: I) + where + I: IntoIterator, + { + if let Some(values) = self.materialize_mut() { + values.extend(iter); + } + } + + /// Convert a uniform run into a materialized buffer if needed, then return + /// the mutable level buffer. Returns `None` when no physical level stream exists. + fn materialize_mut(&mut self) -> Option<&mut Vec> { + match self { + Self::Absent => None, + Self::Materialized(values) => Some(values), + Self::Uniform { value, count } => { + let values = vec![*value; *count]; + *self = Self::Materialized(values); + match self { + Self::Materialized(values) => Some(values), + _ => unreachable!(), + } + } + } + } +} + #[derive(Debug, Clone)] pub(crate) struct ArrayLevels { /// Array's definition levels /// /// Present if `max_def_level != 0` - def_levels: Option>, + def_levels: LevelData, /// Array's optional repetition levels /// /// Present if `max_rep_level != 0` - rep_levels: Option>, + rep_levels: LevelData, /// The corresponding array identifying non-null slices of data /// from the primitive array @@ -807,8 +912,8 @@ impl ArrayLevels { let logical_nulls = array.logical_nulls(); Self { - def_levels: (max_def_level != 0).then(Vec::new), - rep_levels: (max_rep_level != 0).then(Vec::new), + def_levels: LevelData::new(max_def_level != 0), + rep_levels: LevelData::new(max_rep_level != 0), non_null_indices: vec![], max_def_level, max_rep_level, @@ -821,12 +926,12 @@ impl ArrayLevels { &self.array } - pub fn def_levels(&self) -> Option<&[i16]> { - self.def_levels.as_deref() + pub(crate) fn def_level_data(&self) -> &LevelData { + &self.def_levels } - pub fn rep_levels(&self) -> Option<&[i16]> { - self.rep_levels.as_deref() + pub(crate) fn rep_level_data(&self) -> &LevelData { + &self.rep_levels } pub fn non_null_indices(&self) -> &[usize] { @@ -839,12 +944,8 @@ impl ArrayLevels { /// `non_null_indices`. The array is sliced to the range covered by /// those indices, and they are shifted to be relative to the slice. pub(crate) fn slice_for_chunk(&self, chunk: &CdcChunk) -> Self { - let def_levels = self.def_levels.as_ref().map(|levels| { - levels[chunk.level_offset..chunk.level_offset + chunk.num_levels].to_vec() - }); - let rep_levels = self.rep_levels.as_ref().map(|levels| { - levels[chunk.level_offset..chunk.level_offset + chunk.num_levels].to_vec() - }); + let def_levels = self.def_levels.slice(chunk.level_offset, chunk.num_levels); + let rep_levels = self.rep_levels.slice(chunk.level_offset, chunk.num_levels); // Select the non-null indices for this chunk. let nni = &self.non_null_indices[chunk.value_offset..chunk.value_offset + chunk.num_values]; @@ -870,6 +971,14 @@ impl ArrayLevels { logical_nulls, } } + + fn append_def_level_run(&mut self, value: i16, count: usize) { + self.def_levels.append_run(value, count); + } + + fn append_rep_level_run(&mut self, value: i16, count: usize) { + self.rep_levels.append_run(value, count); + } } #[cfg(test)] @@ -920,8 +1029,8 @@ mod tests { assert_eq!(levels.len(), 1); let expected = ArrayLevels { - def_levels: Some(vec![2; 10]), - rep_levels: Some(vec![0, 2, 2, 1, 2, 2, 2, 0, 1, 2]), + def_levels: LevelData::Materialized(vec![2; 10]), + rep_levels: LevelData::Materialized(vec![0, 2, 2, 1, 2, 2, 2, 0, 1, 2]), non_null_indices: vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9], max_def_level: 2, max_rep_level: 2, @@ -941,8 +1050,8 @@ mod tests { assert_eq!(levels.len(), 1); let expected_levels = ArrayLevels { - def_levels: None, - rep_levels: None, + def_levels: LevelData::Absent, + rep_levels: LevelData::Absent, non_null_indices: (0..10).collect(), max_def_level: 0, max_rep_level: 0, @@ -969,8 +1078,8 @@ mod tests { let logical_nulls = array.logical_nulls(); let expected_levels = ArrayLevels { - def_levels: Some(vec![1, 0, 1, 1, 0]), - rep_levels: None, + def_levels: LevelData::Materialized(vec![1, 0, 1, 1, 0]), + rep_levels: LevelData::Absent, non_null_indices: vec![0, 2, 3], max_def_level: 1, max_rep_level: 0, @@ -1004,8 +1113,8 @@ mod tests { assert_eq!(levels.len(), 1); let expected_levels = ArrayLevels { - def_levels: Some(vec![1; 5]), - rep_levels: Some(vec![0; 5]), + def_levels: LevelData::Materialized(vec![1; 5]), + rep_levels: LevelData::Materialized(vec![0; 5]), non_null_indices: (0..5).collect(), max_def_level: 1, max_rep_level: 1, @@ -1038,8 +1147,8 @@ mod tests { assert_eq!(levels.len(), 1); let expected_levels = ArrayLevels { - def_levels: Some(vec![2, 2, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2]), - rep_levels: Some(vec![0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1]), + def_levels: LevelData::Materialized(vec![2, 2, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2]), + rep_levels: LevelData::Materialized(vec![0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1]), non_null_indices: (0..11).collect(), max_def_level: 2, max_rep_level: 1, @@ -1088,8 +1197,8 @@ mod tests { assert_eq!(levels.len(), 1); let expected_levels = ArrayLevels { - def_levels: Some(vec![0, 2, 0, 3, 3, 3, 3, 3, 3, 3]), - rep_levels: Some(vec![0, 0, 0, 0, 1, 1, 1, 0, 1, 1]), + def_levels: LevelData::Materialized(vec![0, 2, 0, 3, 3, 3, 3, 3, 3, 3]), + rep_levels: LevelData::Materialized(vec![0, 0, 0, 0, 1, 1, 1, 0, 1, 1]), non_null_indices: (4..11).collect(), max_def_level: 3, max_rep_level: 1, @@ -1135,10 +1244,10 @@ mod tests { assert_eq!(levels.len(), 1); let expected_levels = ArrayLevels { - def_levels: Some(vec![ + def_levels: LevelData::Materialized(vec![ 5, 5, 5, 5, 1, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, ]), - rep_levels: Some(vec![ + rep_levels: LevelData::Materialized(vec![ 0, 2, 1, 2, 0, 0, 2, 1, 2, 0, 2, 1, 2, 1, 2, 1, 2, 0, 2, 1, 2, 1, 2, ]), non_null_indices: (0..22).collect(), @@ -1177,8 +1286,8 @@ mod tests { assert_eq!(levels.len(), 1); let expected_levels = ArrayLevels { - def_levels: Some(vec![1; 4]), - rep_levels: Some(vec![0; 4]), + def_levels: LevelData::Materialized(vec![1; 4]), + rep_levels: LevelData::Materialized(vec![0; 4]), non_null_indices: (0..4).collect(), max_def_level: 1, max_rep_level: 1, @@ -1210,8 +1319,8 @@ mod tests { assert_eq!(levels.len(), 1); let expected_levels = ArrayLevels { - def_levels: Some(vec![1, 3, 3, 3, 3, 3, 3, 3]), - rep_levels: Some(vec![0, 0, 1, 1, 0, 1, 0, 1]), + def_levels: LevelData::Materialized(vec![1, 3, 3, 3, 3, 3, 3, 3]), + rep_levels: LevelData::Materialized(vec![0, 0, 1, 1, 0, 1, 0, 1]), non_null_indices: (0..7).collect(), max_def_level: 3, max_rep_level: 1, @@ -1259,8 +1368,12 @@ mod tests { assert_eq!(levels.len(), 1); let expected_levels = ArrayLevels { - def_levels: Some(vec![1, 5, 5, 5, 4, 5, 5, 5, 5, 5, 5, 5, 4, 5, 5, 5, 5, 5]), - rep_levels: Some(vec![0, 0, 1, 2, 1, 0, 2, 2, 1, 2, 2, 2, 0, 1, 2, 2, 2, 2]), + def_levels: LevelData::Materialized(vec![ + 1, 5, 5, 5, 4, 5, 5, 5, 5, 5, 5, 5, 4, 5, 5, 5, 5, 5, + ]), + rep_levels: LevelData::Materialized(vec![ + 0, 0, 1, 2, 1, 0, 2, 2, 1, 2, 2, 2, 0, 1, 2, 2, 2, 2, + ]), non_null_indices: (0..15).collect(), max_def_level: 5, max_rep_level: 2, @@ -1300,8 +1413,8 @@ mod tests { let logical_nulls = leaf.logical_nulls(); let expected_levels = ArrayLevels { - def_levels: Some(vec![3, 2, 3, 1, 0, 3]), - rep_levels: None, + def_levels: LevelData::Materialized(vec![3, 2, 3, 1, 0, 3]), + rep_levels: LevelData::Absent, non_null_indices: vec![0, 2, 5], max_def_level: 3, max_rep_level: 0, @@ -1340,8 +1453,8 @@ mod tests { let list_level = &levels[0]; let expected_level = ArrayLevels { - def_levels: Some(vec![0, 3, 3, 3]), - rep_levels: Some(vec![0, 0, 1, 1]), + def_levels: LevelData::Materialized(vec![0, 3, 3, 3]), + rep_levels: LevelData::Materialized(vec![0, 0, 1, 1]), non_null_indices: vec![3, 4, 5], max_def_level: 3, max_rep_level: 1, @@ -1433,8 +1546,8 @@ mod tests { let list_level = &levels[0]; let expected_level = ArrayLevels { - def_levels: None, - rep_levels: None, + def_levels: LevelData::Absent, + rep_levels: LevelData::Absent, non_null_indices: vec![0, 1, 2, 3, 4], max_def_level: 0, max_rep_level: 0, @@ -1448,8 +1561,8 @@ mod tests { let b_logical_nulls = b.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![1, 0, 0, 1, 1]), - rep_levels: None, + def_levels: LevelData::Materialized(vec![1, 0, 0, 1, 1]), + rep_levels: LevelData::Absent, non_null_indices: vec![0, 3, 4], max_def_level: 1, max_rep_level: 0, @@ -1463,8 +1576,8 @@ mod tests { let d_logical_nulls = d.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![1, 1, 1, 2, 1]), - rep_levels: None, + def_levels: LevelData::Materialized(vec![1, 1, 1, 2, 1]), + rep_levels: LevelData::Absent, non_null_indices: vec![3], max_def_level: 2, max_rep_level: 0, @@ -1478,8 +1591,8 @@ mod tests { let f_logical_nulls = f.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![3, 2, 3, 2, 3]), - rep_levels: None, + def_levels: LevelData::Materialized(vec![3, 2, 3, 2, 3]), + rep_levels: LevelData::Absent, non_null_indices: vec![0, 2, 4], max_def_level: 3, max_rep_level: 0, @@ -1586,8 +1699,8 @@ mod tests { let list_level = &levels[0]; let expected_level = ArrayLevels { - def_levels: Some(vec![1; 7]), - rep_levels: Some(vec![0, 1, 0, 1, 0, 1, 1]), + def_levels: LevelData::Materialized(vec![1; 7]), + rep_levels: LevelData::Materialized(vec![0, 1, 0, 1, 0, 1, 1]), non_null_indices: vec![0, 1, 2, 3, 4, 5, 6], max_def_level: 1, max_rep_level: 1, @@ -1601,8 +1714,8 @@ mod tests { let map_values_logical_nulls = map.values().logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![2, 2, 2, 1, 2, 1, 2]), - rep_levels: Some(vec![0, 1, 0, 1, 0, 1, 1]), + def_levels: LevelData::Materialized(vec![2, 2, 2, 1, 2, 1, 2]), + rep_levels: LevelData::Materialized(vec![0, 1, 0, 1, 0, 1, 1]), non_null_indices: vec![0, 1, 2, 4, 6], max_def_level: 2, max_rep_level: 1, @@ -1688,8 +1801,8 @@ mod tests { let logical_nulls = values.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![4, 1, 0, 2, 2, 3, 4]), - rep_levels: Some(vec![0, 0, 0, 0, 1, 0, 0]), + def_levels: LevelData::Materialized(vec![4, 1, 0, 2, 2, 3, 4]), + rep_levels: LevelData::Materialized(vec![0, 0, 0, 0, 1, 0, 0]), non_null_indices: vec![0, 4], max_def_level: 4, max_rep_level: 1, @@ -1730,8 +1843,8 @@ mod tests { let logical_nulls = values.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![4, 4, 3, 2, 0, 4, 4, 0, 1]), - rep_levels: Some(vec![0, 1, 0, 0, 0, 0, 1, 0, 0]), + def_levels: LevelData::Materialized(vec![4, 4, 3, 2, 0, 4, 4, 0, 1]), + rep_levels: LevelData::Materialized(vec![0, 1, 0, 0, 0, 0, 1, 0, 0]), non_null_indices: vec![0, 1, 5, 6], max_def_level: 4, max_rep_level: 1, @@ -1817,8 +1930,8 @@ mod tests { let a1_logical_nulls = a1_values.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![0, 0, 1, 6, 5, 2, 3, 1]), - rep_levels: Some(vec![0, 0, 0, 0, 2, 0, 1, 0]), + def_levels: LevelData::Materialized(vec![0, 0, 1, 6, 5, 2, 3, 1]), + rep_levels: LevelData::Materialized(vec![0, 0, 0, 0, 2, 0, 1, 0]), non_null_indices: vec![1], max_def_level: 6, max_rep_level: 2, @@ -1830,8 +1943,8 @@ mod tests { let a2_logical_nulls = a2_values.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![0, 0, 1, 3, 2, 4, 1]), - rep_levels: Some(vec![0, 0, 0, 0, 0, 1, 0]), + def_levels: LevelData::Materialized(vec![0, 0, 1, 3, 2, 4, 1]), + rep_levels: LevelData::Materialized(vec![0, 0, 0, 0, 0, 1, 0]), non_null_indices: vec![4], max_def_level: 4, max_rep_level: 1, @@ -1870,8 +1983,8 @@ mod tests { let logical_nulls = values.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![0, 0, 3, 3]), - rep_levels: Some(vec![0, 0, 0, 1]), + def_levels: LevelData::Materialized(vec![0, 0, 3, 3]), + rep_levels: LevelData::Materialized(vec![0, 0, 0, 1]), non_null_indices: vec![6, 7], max_def_level: 3, max_rep_level: 1, @@ -2022,8 +2135,8 @@ mod tests { // [[{a: 1}, null], null, [null, null], [{a: null}, {a: 2}]] let values_a_logical_nulls = values_a.logical_nulls(); let expected_a = ArrayLevels { - def_levels: Some(vec![4, 2, 0, 2, 2, 3, 4]), - rep_levels: Some(vec![0, 1, 0, 0, 1, 0, 1]), + def_levels: LevelData::Materialized(vec![4, 2, 0, 2, 2, 3, 4]), + rep_levels: LevelData::Materialized(vec![0, 1, 0, 0, 1, 0, 1]), non_null_indices: vec![0, 7], max_def_level: 4, max_rep_level: 1, @@ -2033,8 +2146,8 @@ mod tests { // [[{b: 2}, null], null, [null, null], [{b: 3}, {b: 4}]] let values_b_logical_nulls = values_b.logical_nulls(); let expected_b = ArrayLevels { - def_levels: Some(vec![3, 2, 0, 2, 2, 3, 3]), - rep_levels: Some(vec![0, 1, 0, 0, 1, 0, 1]), + def_levels: LevelData::Materialized(vec![3, 2, 0, 2, 2, 3, 3]), + rep_levels: LevelData::Materialized(vec![0, 1, 0, 0, 1, 0, 1]), non_null_indices: vec![0, 6, 7], max_def_level: 3, max_rep_level: 1, @@ -2066,8 +2179,8 @@ mod tests { let logical_nulls = values.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![1, 0, 1]), - rep_levels: Some(vec![0, 0, 0]), + def_levels: LevelData::Materialized(vec![1, 0, 1]), + rep_levels: LevelData::Materialized(vec![0, 0, 0]), non_null_indices: vec![], max_def_level: 3, max_rep_level: 1, @@ -2103,8 +2216,8 @@ mod tests { let logical_nulls = values.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![5, 4, 5, 2, 5, 3, 5, 5, 4, 4, 0]), - rep_levels: Some(vec![0, 2, 2, 1, 0, 1, 0, 2, 1, 2, 0]), + def_levels: LevelData::Materialized(vec![5, 4, 5, 2, 5, 3, 5, 5, 4, 4, 0]), + rep_levels: LevelData::Materialized(vec![0, 2, 2, 1, 0, 1, 0, 2, 1, 2, 0]), non_null_indices: vec![0, 2, 3, 4, 5], max_def_level: 5, max_rep_level: 2, @@ -2136,8 +2249,8 @@ mod tests { let logical_nulls = dict.logical_nulls(); let expected_level = ArrayLevels { - def_levels: Some(vec![0, 0, 1, 1]), - rep_levels: None, + def_levels: LevelData::Materialized(vec![0, 0, 1, 1]), + rep_levels: LevelData::Absent, non_null_indices: vec![2, 3], max_def_level: 1, max_rep_level: 0, @@ -2176,8 +2289,8 @@ mod tests { let array: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5, 6])); let logical_nulls = array.logical_nulls(); let levels = ArrayLevels { - def_levels: None, - rep_levels: None, + def_levels: LevelData::Absent, + rep_levels: LevelData::Absent, non_null_indices: vec![0, 1, 2, 3, 4, 5], max_def_level: 0, max_rep_level: 0, @@ -2190,8 +2303,8 @@ mod tests { value_offset: 2, num_values: 3, }); - assert!(sliced.def_levels.is_none()); - assert!(sliced.rep_levels.is_none()); + assert!(matches!(sliced.def_levels, LevelData::Absent)); + assert!(matches!(sliced.rep_levels, LevelData::Absent)); assert_eq!(sliced.non_null_indices, vec![0, 1, 2]); assert_eq!(sliced.array.len(), 3); @@ -2210,8 +2323,8 @@ mod tests { ])); let logical_nulls = array.logical_nulls(); let levels = ArrayLevels { - def_levels: Some(vec![1, 0, 1, 0, 1, 1]), - rep_levels: None, + def_levels: LevelData::Materialized(vec![1, 0, 1, 0, 1, 1]), + rep_levels: LevelData::Absent, non_null_indices: vec![0, 2, 4, 5], max_def_level: 1, max_rep_level: 0, @@ -2224,8 +2337,8 @@ mod tests { value_offset: 1, num_values: 1, }); - assert_eq!(sliced.def_levels, Some(vec![0, 1, 0])); - assert!(sliced.rep_levels.is_none()); + assert_eq!(sliced.def_levels, LevelData::Materialized(vec![0, 1, 0])); + assert!(matches!(sliced.rep_levels, LevelData::Absent)); assert_eq!(sliced.non_null_indices, vec![0]); // [2] shifted by -2 (nni[0]) assert_eq!(sliced.array.len(), 1); } @@ -2264,8 +2377,8 @@ mod tests { ])); let logical_nulls = array.logical_nulls(); let levels = ArrayLevels { - def_levels: Some(vec![3, 0, 3, 2, 0, 3, 3]), - rep_levels: Some(vec![0, 0, 0, 1, 0, 0, 1]), + def_levels: LevelData::Materialized(vec![3, 0, 3, 2, 0, 3, 3]), + rep_levels: LevelData::Materialized(vec![0, 0, 0, 1, 0, 0, 1]), non_null_indices: vec![0, 3, 8, 9], max_def_level: 3, max_rep_level: 1, @@ -2310,8 +2423,8 @@ mod tests { let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None, None, Some(4)])); let logical_nulls = array.logical_nulls(); let levels = ArrayLevels { - def_levels: Some(vec![1, 0, 0, 1]), - rep_levels: None, + def_levels: LevelData::Materialized(vec![1, 0, 0, 1]), + rep_levels: LevelData::Absent, non_null_indices: vec![0, 3], max_def_level: 1, max_rep_level: 0, @@ -2325,7 +2438,7 @@ mod tests { value_offset: 1, num_values: 0, }); - assert_eq!(sliced.def_levels, Some(vec![0, 0])); + assert_eq!(sliced.def_levels, LevelData::Materialized(vec![0, 0])); assert_eq!(sliced.non_null_indices, Vec::::new()); assert_eq!(sliced.array.len(), 0); } diff --git a/parquet/src/arrow/arrow_writer/mod.rs b/parquet/src/arrow/arrow_writer/mod.rs index 641b81257a91..04a798391429 100644 --- a/parquet/src/arrow/arrow_writer/mod.rs +++ b/parquet/src/arrow/arrow_writer/mod.rs @@ -920,8 +920,11 @@ impl ArrowColumnWriter { chunker: &mut ContentDefinedChunker, ) -> Result<()> { let levels = &col.0; - let chunks = - chunker.get_arrow_chunks(levels.def_levels(), levels.rep_levels(), levels.array())?; + let chunks = chunker.get_arrow_chunks( + levels.def_level_data().as_ref(), + levels.rep_level_data().as_ref(), + levels.array(), + )?; let num_chunks = chunks.len(); for (i, chunk) in chunks.iter().enumerate() { @@ -1384,10 +1387,15 @@ fn write_leaf( } ColumnWriter::BoolColumnWriter(typed) => { let array = column.as_boolean(); - typed.write_batch( - get_bool_array_slice(array, indices).as_slice(), - levels.def_levels(), - levels.rep_levels(), + let values = get_bool_array_slice(array, indices); + typed.write_batch_internal( + values.as_slice(), + None, + levels.def_level_data().as_ref(), + levels.rep_level_data().as_ref(), + None, + None, + None, ) } ColumnWriter::Int64ColumnWriter(typed) => { @@ -1538,7 +1546,15 @@ fn write_leaf( )); } }; - typed.write_batch(bytes.as_slice(), levels.def_levels(), levels.rep_levels()) + typed.write_batch_internal( + bytes.as_slice(), + None, + levels.def_level_data().as_ref(), + levels.rep_level_data().as_ref(), + None, + None, + None, + ) } } } @@ -1551,8 +1567,8 @@ fn write_primitive( writer.write_batch_internal( values, Some(levels.non_null_indices()), - levels.def_levels(), - levels.rep_levels(), + levels.def_level_data().as_ref(), + levels.rep_level_data().as_ref(), None, None, None, @@ -2782,7 +2798,7 @@ mod tests { .set_bloom_filter_enabled(bloom_filter) .set_bloom_filter_position(bloom_filter_position); if let Some(ndv) = bloom_filter_ndv { - builder = builder.set_bloom_filter_ndv(ndv); + builder = builder.set_bloom_filter_max_ndv(ndv); } let props = builder.build(); @@ -4580,6 +4596,44 @@ mod tests { } } + #[test] + fn test_arrow_writer_skip_path_in_schema() { + let batch_schema = Schema::new(vec![Field::new("int32", DataType::Int32, false)]); + let file_schema = Arc::new(batch_schema.clone()); + + let batch = RecordBatch::try_new( + Arc::new(batch_schema), + vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4])) as _], + ) + .unwrap(); + + // default options should still write path_in_schema + let skip_options = ArrowWriterOptions::new(); + + let mut buf = Vec::with_capacity(1024); + let mut writer = + ArrowWriter::try_new_with_options(&mut buf, file_schema.clone(), skip_options).unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + // override to not write path_in_schema + let skip_options = ArrowWriterOptions::new().with_properties( + WriterProperties::builder() + .set_write_path_in_schema(false) + .build(), + ); + + let mut buf2 = Vec::with_capacity(1024); + let mut writer = + ArrowWriter::try_new_with_options(&mut buf2, file_schema.clone(), skip_options) + .unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + + // buf2 should be a bit smaller due to lack of path_in_schema + assert!(buf.len() > buf2.len()); + } + #[test] fn mismatched_schemas() { let batch_schema = Schema::new(vec![Field::new("count", DataType::Int32, false)]); diff --git a/parquet/src/arrow/push_decoder/mod.rs b/parquet/src/arrow/push_decoder/mod.rs index 4c667e534366..0fe67195a65d 100644 --- a/parquet/src/arrow/push_decoder/mod.rs +++ b/parquet/src/arrow/push_decoder/mod.rs @@ -22,15 +22,17 @@ mod reader_builder; mod remaining; use crate::DecodeResult; +use crate::arrow::ProjectionMask; use crate::arrow::arrow_reader::{ ArrowReaderBuilder, ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReader, + RowFilter, RowSelectionPolicy, }; use crate::errors::ParquetError; use crate::file::metadata::ParquetMetaData; use crate::util::push_buffers::PushBuffers; use arrow_array::RecordBatch; use bytes::Bytes; -use reader_builder::RowGroupReaderBuilder; +use reader_builder::{RowBudget, RowGroupReaderBuilder}; use remaining::RemainingRowGroups; use std::ops::Range; use std::sync::Arc; @@ -181,6 +183,9 @@ impl ParquetPushDecoderBuilder { // If no row groups were specified, read all of them let row_groups = row_groups.unwrap_or_else(|| (0..parquet_metadata.num_row_groups()).collect()); + let has_predicates = filter + .as_ref() + .is_some_and(|filter| !filter.predicates.is_empty()); // Prepare to build RowGroup readers let file_len = 0; // not used in push decoder @@ -191,8 +196,6 @@ impl ParquetPushDecoderBuilder { Arc::clone(&parquet_metadata), fields, filter, - limit, - offset, metrics, max_predicate_cache_size, buffers, @@ -204,6 +207,8 @@ impl ParquetPushDecoderBuilder { parquet_metadata, row_groups, selection, + RowBudget::new(offset, limit), + has_predicates, row_group_reader_builder, ); @@ -374,6 +379,100 @@ impl ParquetPushDecoder { pub fn clear_all_ranges(&mut self) { self.state.clear_all_ranges(); } + + /// True iff the decoder is at a row-group boundary and a + /// [`Self::swap_strategy`] call would succeed. + /// + /// A boundary is "between row groups": the previous row group's + /// [`ParquetRecordBatchReader`] has been fully extracted (via + /// [`Self::try_next_reader`]) or fully drained (via [`Self::try_decode`]), + /// and the next row group has not yet been planned. While + /// [`Self::try_decode`] is iterating an active row group's reader this + /// returns `false`; with [`Self::try_next_reader`] there is a clean + /// window between two consecutive returns where this is `true`. + pub fn can_swap_strategy(&self) -> bool { + self.state.can_swap_strategy() + } + + /// Number of row groups left to decode after the one currently in flight. + /// Useful as a "should I bother computing a new strategy?" signal. + pub fn row_groups_remaining(&self) -> usize { + self.state.row_groups_remaining() + } + + /// Replace projection / row filter / row selection policy for subsequent + /// row groups. + /// + /// Returns `Err(ParquetError::General)` when called outside a row-group + /// boundary; check [`Self::can_swap_strategy`] first. + /// + /// The decoder's internal `PushBuffers` are preserved across the swap. + /// Bytes that have already been fetched for columns the new strategy + /// still needs will not be re-requested. Bytes for columns the new + /// strategy no longer needs remain buffered until + /// [`Self::clear_all_ranges`] is called or the decoder is dropped. + /// + /// Limit, offset, batch size, metadata, fields, and predicate-cache size + /// are unchanged by a swap. + pub fn swap_strategy(&mut self, swap: StrategySwap) -> Result<(), ParquetError> { + self.state.swap_strategy(swap) + } +} + +/// Description of a strategy swap to apply via +/// [`ParquetPushDecoder::swap_strategy`]. +/// +/// Each field is `Option`-wrapped so callers only override what they intend +/// to change. Fields left as `None` carry their previous value through the +/// swap. +/// +/// [`Self::filter`] is doubly-`Option`-wrapped on purpose: the outer +/// `Option` is "do you want to change the filter?", the inner is "set or +/// clear?". `Some(None)` clears any previously-installed filter. +#[derive(Debug, Default)] +#[non_exhaustive] +pub struct StrategySwap { + /// New projection mask. `None` means "leave the projection alone". + pub projection: Option, + /// New row filter. Outer `None` means "leave the filter alone"; + /// `Some(None)` clears the filter; `Some(Some(filter))` installs a new + /// one. + pub filter: Option>, + /// New row selection policy. `None` means "leave the policy alone". + pub row_selection_policy: Option, +} + +impl StrategySwap { + /// Empty swap. No fields will be modified — chain `with_*` setters to + /// configure. + pub fn new() -> Self { + Self::default() + } + + /// Set a new projection mask for subsequent row groups. + pub fn with_projection(mut self, projection: ProjectionMask) -> Self { + self.projection = Some(projection); + self + } + + /// Install a new row filter (or `None` to clear) for subsequent row + /// groups. + pub fn with_filter(mut self, filter: Option) -> Self { + self.filter = Some(filter); + self + } + + /// Set a new row selection policy for subsequent row groups. + pub fn with_row_selection_policy(mut self, policy: RowSelectionPolicy) -> Self { + self.row_selection_policy = Some(policy); + self + } + + /// True iff every field is `None`. A no-op swap returns successfully but + /// has no effect. + pub fn is_empty(&self) -> bool { + self.projection.is_none() && self.filter.is_none() && self.row_selection_policy.is_none() + } } /// Internal state machine for the [`ParquetPushDecoder`] @@ -596,6 +695,75 @@ impl ParquetDecoderState { ParquetDecoderState::Finished => {} } } + + fn can_swap_strategy(&self) -> bool { + match self { + ParquetDecoderState::ReadingRowGroup { + remaining_row_groups, + } => remaining_row_groups.is_at_row_group_boundary(), + // Mid-row-group: the active reader holds an `ArrayReader` and + // `ReadPlan` keyed to the *current* projection/filter; swapping + // would require throwing that work away. + ParquetDecoderState::DecodingRowGroup { .. } => false, + ParquetDecoderState::Finished => false, + } + } + + fn row_groups_remaining(&self) -> usize { + match self { + ParquetDecoderState::ReadingRowGroup { + remaining_row_groups, + } => remaining_row_groups.row_groups_remaining(), + ParquetDecoderState::DecodingRowGroup { + remaining_row_groups, + .. + } => remaining_row_groups.row_groups_remaining(), + ParquetDecoderState::Finished => 0, + } + } + + fn swap_strategy(&mut self, swap: StrategySwap) -> Result<(), ParquetError> { + if swap.is_empty() { + return Ok(()); + } + let remaining = match self { + ParquetDecoderState::ReadingRowGroup { + remaining_row_groups, + } => remaining_row_groups, + ParquetDecoderState::DecodingRowGroup { .. } => { + return Err(ParquetError::General( + "swap_strategy called while a row group is being decoded; \ + check can_swap_strategy() first" + .to_string(), + )); + } + ParquetDecoderState::Finished => { + return Err(ParquetError::General( + "swap_strategy called on a finished decoder".to_string(), + )); + } + }; + if !remaining.is_at_row_group_boundary() { + return Err(ParquetError::General( + "swap_strategy called mid-row-group; check can_swap_strategy() first".to_string(), + )); + } + let StrategySwap { + projection, + filter, + row_selection_policy, + } = swap; + if let Some(projection) = projection { + remaining.set_projection(projection)?; + } + if let Some(filter) = filter { + remaining.set_filter(filter)?; + } + if let Some(policy) = row_selection_policy { + remaining.set_row_selection_policy(policy)?; + } + Ok(()) + } } #[cfg(test)] @@ -1402,6 +1570,28 @@ mod test { expect_finished(decoder.try_decode()); } + #[test] + fn test_decoder_try_next_reader_offset_limit() { + let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()) + .unwrap() + .with_offset(225) + .with_limit(20) + .build() + .unwrap(); + + let ranges = expect_needs_data(decoder.try_next_reader()); + push_ranges_to_decoder(&mut decoder, ranges); + + let reader = expect_data(decoder.try_next_reader()); + let batches = reader + .map(|batch| batch.expect("expected decoded batch")) + .collect::>(); + let output = concat_batches(&TEST_BATCH.schema(), &batches).unwrap(); + assert_eq!(output, TEST_BATCH.slice(225, 20)); + + expect_finished(decoder.try_next_reader()); + } + #[test] fn test_decoder_row_group_selection() { // take only the second row group @@ -1451,6 +1641,326 @@ mod test { expect_finished(decoder.try_decode()); } + /// `swap_strategy` between row groups installs a new filter for the + /// next row group while leaving the just-decoded row group's results + /// untouched. + /// + /// Adaptive callers should drive the decoder with `try_next_reader` + /// rather than `try_decode`: `try_next_reader` returns once per row + /// group, giving the caller a clean window between two consecutive + /// returns to inspect stats and call `swap_strategy`. `try_decode` + /// barrels through row-group boundaries and is unsuitable for in-flight + /// strategy changes. + #[test] + fn test_swap_strategy_installs_filter_between_row_groups() { + let metadata = test_file_parquet_metadata(); + let schema_descr = metadata.file_metadata().schema_descr_ptr(); + + let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(metadata) + .unwrap() + .with_batch_size(1024) + .build() + .unwrap(); + + // Prefetch the entire file so we don't have to interleave I/O with + // the swap. PushBuffers carries the bytes through the swap. + decoder + .push_range(test_file_range(), TEST_FILE_DATA.clone()) + .unwrap(); + + // Reader for row group 0 — no filter. + let reader0 = expect_data(decoder.try_next_reader()); + let batches0: Vec<_> = reader0.collect::>().unwrap(); + let batch0 = concat_batches(&TEST_BATCH.schema(), &batches0).unwrap(); + assert_eq!(batch0.num_rows(), 200); + assert_eq!(batch0, TEST_BATCH.slice(0, 200)); + + // We're between row groups now. Swap in a filter on column "a". + assert!(decoder.can_swap_strategy()); + let filter = + ArrowPredicateFn::new(ProjectionMask::columns(&schema_descr, ["a"]), |batch| { + gt(batch.column(0), &Int64Array::new_scalar(250)) + }); + decoder + .swap_strategy( + StrategySwap::new().with_filter(Some(RowFilter::new(vec![Box::new(filter)]))), + ) + .unwrap(); + + // Reader for row group 1 — filter applied. Column "a" in RG1 has + // values 200..399; `a > 250` keeps 251..399 = 149 rows. + let reader1 = expect_data(decoder.try_next_reader()); + let batches1: Vec<_> = reader1.collect::>().unwrap(); + let batch1 = concat_batches(&TEST_BATCH.schema(), &batches1).unwrap(); + let expected1 = TEST_BATCH.slice(251, 149); + assert_eq!(batch1, expected1); + expect_finished(decoder.try_next_reader()); + } + + /// `swap_strategy` is rejected while a row group's reader is being + /// drained, and accepted again the moment we cross the boundary. + #[test] + fn test_swap_strategy_rejected_mid_row_group() { + let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()) + .unwrap() + .with_batch_size(50) + .build() + .unwrap(); + + decoder + .push_range(test_file_range(), TEST_FILE_DATA.clone()) + .unwrap(); + + // After getting the first batch, we're inside `DecodingRowGroup`: + // an active reader is still alive. Mid-reader is not a swap point. + let _ = expect_data(decoder.try_decode()); + assert!(!decoder.can_swap_strategy()); + + // Trying to swap mid-row-group is an error. + let err = decoder + .swap_strategy( + StrategySwap::new().with_row_selection_policy(super::RowSelectionPolicy::Mask), + ) + .unwrap_err(); + let err_msg = format!("{err}"); + assert!( + err_msg.contains("can_swap_strategy"), + "unexpected error: {err_msg}" + ); + + // Empty swap is a no-op even mid-row-group. + decoder.swap_strategy(StrategySwap::new()).unwrap(); + } + + /// `try_next_reader` hands the active reader off to the caller and + /// transitions the decoder back to `ReadingRowGroup` — so the caller + /// can call `swap_strategy` even while still iterating the returned + /// reader. (The handed-off reader has no link back to the decoder's + /// projection/filter; it has its own `ArrayReader` and `ReadPlan`.) + #[test] + fn test_swap_strategy_allowed_while_iterating_handed_off_reader() { + let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(test_file_parquet_metadata()) + .unwrap() + .with_batch_size(1024) + .build() + .unwrap(); + decoder + .push_range(test_file_range(), TEST_FILE_DATA.clone()) + .unwrap(); + + let reader0 = expect_data(decoder.try_next_reader()); + // Decoder no longer owns the reader, so it considers itself + // "between row groups" — swap is allowed even though the caller + // hasn't iterated `reader0` yet. + assert!(decoder.can_swap_strategy()); + // Iterating `reader0` is independent of the decoder's state. + let batches: Vec<_> = reader0.collect::>().unwrap(); + let batch0 = concat_batches(&TEST_BATCH.schema(), &batches).unwrap(); + assert_eq!(batch0, TEST_BATCH.slice(0, 200)); + } + + /// Bytes already buffered for a column survive a projection narrowing — + /// the next row group should not request any new data for the column + /// that was *already* in the projection. + #[test] + fn test_swap_strategy_preserves_buffered_bytes() { + let metadata = test_file_parquet_metadata(); + let schema_descr = metadata.file_metadata().schema_descr_ptr(); + + let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(metadata) + .unwrap() + .with_batch_size(1024) + .build() + .unwrap(); + + // Prefetch the whole file once. + decoder + .push_range(test_file_range(), TEST_FILE_DATA.clone()) + .unwrap(); + assert_eq!(decoder.buffered_bytes(), test_file_len()); + + // Drain RG0 with full projection via try_next_reader. + let reader0 = expect_data(decoder.try_next_reader()); + let _: Vec<_> = reader0.collect::>().unwrap(); + + // Narrow projection to just column "a" for RG1. + assert!(decoder.can_swap_strategy()); + decoder + .swap_strategy( + StrategySwap::new().with_projection(ProjectionMask::columns(&schema_descr, ["a"])), + ) + .unwrap(); + + // Column "a" bytes for RG1 were part of the original wide + // projection's prefetch; we should not need additional data. + let reader1 = expect_data(decoder.try_next_reader()); + let batches1: Vec<_> = reader1.collect::>().unwrap(); + // Schema is now just "a"; RG1 has values 200..399. + let expected1 = TEST_BATCH.slice(200, 200).project(&[0]).unwrap(); + let batch1 = concat_batches(&batches1[0].schema(), &batches1).unwrap(); + assert_eq!(batch1, expected1); + expect_finished(decoder.try_next_reader()); + } + + /// Mirror of [`Self::test_swap_strategy_preserves_buffered_bytes`] in the + /// opposite direction: start with a narrow projection, drain RG0, then + /// widen the projection to add columns "b" and "c". Because the entire + /// file was prefetched, bytes for the newly-added columns are already + /// buffered and the next row group decodes without another `NeedsData`. + #[test] + fn test_swap_strategy_expands_projection_between_row_groups() { + let metadata = test_file_parquet_metadata(); + let schema_descr = metadata.file_metadata().schema_descr_ptr(); + + let builder = ParquetPushDecoderBuilder::try_new_decoder(metadata) + .unwrap() + .with_batch_size(1024); + let mut decoder = builder + .with_projection(ProjectionMask::columns(&schema_descr, ["a"])) + .build() + .unwrap(); + + // Prefetch the whole file up front. + decoder + .push_range(test_file_range(), TEST_FILE_DATA.clone()) + .unwrap(); + + // RG0 with narrow projection — just column "a". + let reader0 = expect_data(decoder.try_next_reader()); + let batches0: Vec<_> = reader0.collect::>().unwrap(); + let batch0 = concat_batches(&batches0[0].schema(), &batches0).unwrap(); + let expected0 = TEST_BATCH.slice(0, 200).project(&[0]).unwrap(); + assert_eq!(batch0, expected0); + + // At the boundary, widen to all three columns. + assert!(decoder.can_swap_strategy()); + decoder + .swap_strategy(StrategySwap::new().with_projection(ProjectionMask::columns( + &schema_descr, + ["a", "b", "c"], + ))) + .unwrap(); + + // Bytes for "b" and "c" in RG1 were part of the prefetch, so no + // further data should be requested. + let reader1 = expect_data(decoder.try_next_reader()); + let batches1: Vec<_> = reader1.collect::>().unwrap(); + let batch1 = concat_batches(&TEST_BATCH.schema(), &batches1).unwrap(); + assert_eq!(batch1, TEST_BATCH.slice(200, 200)); + expect_finished(decoder.try_next_reader()); + } + + /// Drive the decoder incrementally (no prefetch). Start with a narrow + /// projection, drain RG0 by satisfying its `NeedsData`, then widen the + /// projection to add columns "b" and "c". The post-swap `NeedsData` + /// for RG1 must request bytes for *all three* columns, not just the + /// originally-projected "a". The expected ranges are hardcoded + /// because `TEST_BATCH` and the writer settings are static; this + /// pins the layout cleanly without a parallel reference decoder. + #[test] + fn test_swap_strategy_expand_projection_requests_new_bytes() { + let metadata = test_file_parquet_metadata(); + let schema_descr = metadata.file_metadata().schema_descr_ptr(); + + let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(metadata) + .unwrap() + .with_batch_size(1024) + .with_projection(ProjectionMask::columns(&schema_descr, ["a"])) + .build() + .unwrap(); + + // RG0: incrementally satisfy the narrow request. + let ranges_rg0 = expect_needs_data(decoder.try_next_reader()); + // For a narrow ("a"-only) projection, RG0 is a single contiguous range. + assert_eq!(ranges_rg0, vec![4..1860]); + push_ranges_to_decoder(&mut decoder, ranges_rg0); + + let reader0 = expect_data(decoder.try_next_reader()); + let batches0: Vec<_> = reader0.collect::>().unwrap(); + let batch0 = concat_batches(&batches0[0].schema(), &batches0).unwrap(); + let expected0 = TEST_BATCH.slice(0, 200).project(&[0]).unwrap(); + assert_eq!(batch0, expected0); + // RG0's "a" pages have been consumed; no surviving buffer. + assert_eq!(decoder.buffered_bytes(), 0); + + // Widen the projection at the boundary. + assert!(decoder.can_swap_strategy()); + decoder + .swap_strategy(StrategySwap::new().with_projection(ProjectionMask::columns( + &schema_descr, + ["a", "b", "c"], + ))) + .unwrap(); + + // Post-swap RG1 request covers RG1's "a", "b", and "c" column + // chunks: ~1.8KiB each for "a" and "b", ~7.5KiB for the + // StringView column "c". Total 11168 bytes — far more than the + // 1856 bytes the narrow projection asked for in RG0. + let ranges_rg1 = expect_needs_data(decoder.try_next_reader()); + assert_eq!( + ranges_rg1, + vec![11062..12918, 12918..14774, 14774..22230] + ); + push_ranges_to_decoder(&mut decoder, ranges_rg1); + + let reader1 = expect_data(decoder.try_next_reader()); + let batches1: Vec<_> = reader1.collect::>().unwrap(); + let batch1 = concat_batches(&TEST_BATCH.schema(), &batches1).unwrap(); + assert_eq!(batch1, TEST_BATCH.slice(200, 200)); + expect_finished(decoder.try_next_reader()); + } + + /// Drive the decoder incrementally (no prefetch). Start with the full + /// projection, drain RG0 by satisfying its `NeedsData`, then narrow + /// the projection to just column "a". The post-swap `NeedsData` for + /// RG1 must request bytes for *only* column "a" — the same single + /// range a narrow projection would have asked for, not the three + /// ranges a wide projection would. + #[test] + fn test_swap_strategy_narrow_projection_skips_unneeded_bytes() { + let metadata = test_file_parquet_metadata(); + let schema_descr = metadata.file_metadata().schema_descr_ptr(); + + let mut decoder = ParquetPushDecoderBuilder::try_new_decoder(metadata) + .unwrap() + .with_batch_size(1024) + .build() + .unwrap(); + + // RG0 with default (full) projection — three columns; three ranges. + let ranges_rg0 = expect_needs_data(decoder.try_next_reader()); + assert_eq!(ranges_rg0, vec![4..1860, 1860..3716, 3716..11062]); + push_ranges_to_decoder(&mut decoder, ranges_rg0); + + let reader0 = expect_data(decoder.try_next_reader()); + let batches0: Vec<_> = reader0.collect::>().unwrap(); + let batch0 = concat_batches(&TEST_BATCH.schema(), &batches0).unwrap(); + assert_eq!(batch0, TEST_BATCH.slice(0, 200)); + assert_eq!(decoder.buffered_bytes(), 0); + + // Narrow the projection at the boundary. + assert!(decoder.can_swap_strategy()); + decoder + .swap_strategy( + StrategySwap::new().with_projection(ProjectionMask::columns(&schema_descr, ["a"])), + ) + .unwrap(); + + // Post-swap RG1 request asks for column "a" only — a single + // 1856-byte range, not the three column-chunk ranges the wide + // projection would have produced. + let ranges_rg1 = expect_needs_data(decoder.try_next_reader()); + assert_eq!(ranges_rg1, vec![11062..12918]); + push_ranges_to_decoder(&mut decoder, ranges_rg1); + + let reader1 = expect_data(decoder.try_next_reader()); + let batches1: Vec<_> = reader1.collect::>().unwrap(); + let batch1 = concat_batches(&batches1[0].schema(), &batches1).unwrap(); + let expected1 = TEST_BATCH.slice(200, 200).project(&[0]).unwrap(); + assert_eq!(batch1, expected1); + expect_finished(decoder.try_next_reader()); + } + /// Returns a batch with 400 rows, with 3 columns: "a", "b", "c" /// /// Note c is a different types (so the data page sizes will be different) diff --git a/parquet/src/arrow/push_decoder/reader_builder/mod.rs b/parquet/src/arrow/push_decoder/reader_builder/mod.rs index 60e50d29524e..d9d301161d61 100644 --- a/parquet/src/arrow/push_decoder/reader_builder/mod.rs +++ b/parquet/src/arrow/push_decoder/reader_builder/mod.rs @@ -18,7 +18,6 @@ mod data; mod filter; -use crate::DecodeResult; use crate::arrow::ProjectionMask; use crate::arrow::array_reader::{ArrayReaderBuilder, CacheOptions, RowGroupCache}; use crate::arrow::arrow_reader::metrics::ArrowReaderMetrics; @@ -42,12 +41,13 @@ use filter::FilterInfo; use std::ops::Range; use std::sync::{Arc, RwLock}; -/// The current row group being read and the read plan +/// The current row group being read, its read plan, and its offset/limit budget. #[derive(Debug)] struct RowGroupInfo { row_group_idx: usize, row_count: usize, plan_builder: ReadPlanBuilder, + budget: RowBudget, } /// This is the inner state machine for reading a single row group. @@ -88,6 +88,107 @@ enum RowGroupDecoderState { Finished, } +/// Running offset/limit budget shared across row groups. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(crate) struct RowBudget { + offset: Option, + limit: Option, +} + +impl RowBudget { + pub(crate) fn new(offset: Option, limit: Option) -> Self { + Self { offset, limit } + } + + pub(crate) fn is_exhausted(self) -> bool { + matches!(self.limit, Some(0)) + } + + /// Returns how many selected rows remain after applying this budget. + pub(crate) fn rows_after(self, rows_before_budget: usize) -> usize { + let rows_after_offset = rows_before_budget.saturating_sub(self.offset.unwrap_or(0)); + match self.limit { + Some(limit) => rows_after_offset.min(limit), + None => rows_after_offset, + } + } + + /// Returns the number of selected rows needed before applying the offset. + fn selected_row_limit(self) -> Option { + self.limit + .map(|limit| limit.saturating_add(self.offset.unwrap_or(0))) + } + + fn apply_to_plan(self, plan_builder: ReadPlanBuilder, row_count: usize) -> BudgetedReadPlan { + let rows_before_budget = plan_builder.num_rows_selected().unwrap_or(row_count); + let plan_builder = plan_builder + .limited(row_count) + .with_offset(self.offset) + .with_limit(self.limit) + .build_limited(); + let rows_after_budget = self.rows_after(rows_before_budget); + + BudgetedReadPlan { + plan_builder, + rows_before_budget, + rows_after_budget, + remaining_budget: self.advance(rows_before_budget, rows_after_budget), + } + } + + /// Advance the budget past one row group. + /// + /// `rows_before_budget` is the number of rows selected before applying the + /// budget, and `rows_after_budget` is the number retained for output from + /// this row group. + pub(crate) fn advance(mut self, rows_before_budget: usize, rows_after_budget: usize) -> Self { + if let Some(offset) = &mut self.offset { + // Reduction is either because of offset or limit, as limit is applied + // after offset has been "exhausted" can just use saturating sub here. + *offset = offset.saturating_sub(rows_before_budget - rows_after_budget); + } + + if rows_after_budget != 0 { + if let Some(limit) = &mut self.limit { + *limit -= rows_after_budget; + } + } + + self + } +} + +#[derive(Debug)] +struct BudgetedReadPlan { + /// Read plan after applying this row group's share of the offset/limit budget. + plan_builder: ReadPlanBuilder, + /// Number of rows selected by row selection and predicates before applying + /// this row group's offset/limit budget. + rows_before_budget: usize, + /// Number of selected rows that remain to be read after applying this row + /// group's offset/limit budget. + rows_after_budget: usize, + /// Budget remaining for later row groups. + remaining_budget: RowBudget, +} + +#[derive(Debug)] +pub(crate) enum RowGroupBuildResult { + /// The active row group is complete without producing a reader. + Finished { + /// Budget remaining after applying this row group's selection. + remaining_budget: RowBudget, + }, + /// More bytes are needed before the active row group can make progress. + NeedsData(Vec>), + /// The active row group produced a reader. + Data { + batch_reader: ParquetRecordBatchReader, + /// Budget remaining after applying this row group's selection. + remaining_budget: RowBudget, + }, +} + /// Result of a state transition #[derive(Debug)] struct NextState { @@ -96,7 +197,7 @@ struct NextState { /// /// * `Some`: the processing should stop and return the result /// * `None`: processing should continue - result: Option>, + result: Option, } impl NextState { @@ -111,10 +212,7 @@ impl NextState { } /// Create a NextState with a result that should be returned - fn result( - next_state: RowGroupDecoderState, - result: DecodeResult, - ) -> Self { + fn result(next_state: RowGroupDecoderState, result: RowGroupBuildResult) -> Self { Self { next_state, result: Some(result), @@ -144,12 +242,6 @@ pub(crate) struct RowGroupReaderBuilder { /// Optional filter filter: Option, - /// Limit to apply to remaining row groups (decremented as rows are read) - limit: Option, - - /// Offset to apply to remaining row groups (decremented as rows are read) - offset: Option, - /// The size in bytes of the predicate cache to use /// /// See [`RowGroupCache`] for details. @@ -180,8 +272,6 @@ impl RowGroupReaderBuilder { metadata: Arc, fields: Option>, filter: Option, - limit: Option, - offset: Option, metrics: ArrowReaderMetrics, max_predicate_cache_size: usize, buffers: PushBuffers, @@ -193,8 +283,6 @@ impl RowGroupReaderBuilder { metadata, fields, filter, - limit, - offset, metrics, max_predicate_cache_size, row_selection_policy, @@ -208,6 +296,63 @@ impl RowGroupReaderBuilder { self.buffers.push_ranges(ranges, buffers); } + /// True iff the inner state is `Finished`. This is the only state in + /// which it is safe to swap projection / filter / row selection policy + /// because no `RowGroupInfo`, `FilterInfo`, or in-flight `DataRequest` + /// is referencing the previous values. + pub(crate) fn is_finished(&self) -> bool { + matches!(self.state, Some(RowGroupDecoderState::Finished)) + } + + /// Replace the projection mask used for subsequent row groups. + /// + /// Must only be called when [`Self::is_finished`]. `PushBuffers` are + /// preserved; bytes already fetched for columns that survive into the + /// new mask are reused. + pub(crate) fn set_projection( + &mut self, + projection: ProjectionMask, + ) -> Result<(), ParquetError> { + if !self.is_finished() { + return Err(ParquetError::General( + "RowGroupReaderBuilder::set_projection: state must be Finished".to_string(), + )); + } + self.projection = projection; + Ok(()) + } + + /// Replace the row filter used for subsequent row groups. + /// + /// Must only be called when [`Self::is_finished`]. Pass `None` to clear + /// the filter, `Some(filter)` to install a new one. + pub(crate) fn set_filter(&mut self, filter: Option) -> Result<(), ParquetError> { + if !self.is_finished() { + return Err(ParquetError::General( + "RowGroupReaderBuilder::set_filter: state must be Finished".to_string(), + )); + } + self.filter = filter; + Ok(()) + } + + /// Replace the row selection policy used for subsequent row groups. + /// + /// Must only be called when [`Self::is_finished`]. + pub(crate) fn set_row_selection_policy( + &mut self, + policy: RowSelectionPolicy, + ) -> Result<(), ParquetError> { + if !self.is_finished() { + return Err(ParquetError::General( + "RowGroupReaderBuilder::set_row_selection_policy: state must be Finished" + .to_string(), + )); + } + self.row_selection_policy = policy; + Ok(()) + } + /// Returns the total number of buffered bytes available pub fn buffered_bytes(&self) -> u64 { self.buffers.buffered_bytes() @@ -233,12 +378,18 @@ impl RowGroupReaderBuilder { }) } + /// Returns true if this builder is currently decoding a row group. + pub(crate) fn has_active_row_group(&self) -> bool { + !matches!(self.state, Some(RowGroupDecoderState::Finished)) + } + /// Setup this reader to read the next row group pub(crate) fn next_row_group( &mut self, row_group_idx: usize, row_count: usize, selection: Option, + budget: RowBudget, ) -> Result<(), ParquetError> { let state = self.take_state()?; if !matches!(state, RowGroupDecoderState::Finished) { @@ -254,22 +405,20 @@ impl RowGroupReaderBuilder { row_group_idx, row_count, plan_builder, + budget, }; self.state = Some(RowGroupDecoderState::Start { row_group_info }); Ok(()) } - /// Try to build the next `ParquetRecordBatchReader` from this RowGroupReader. - /// - /// If more data is needed, returns [`DecodeResult::NeedsData`] with the - /// ranges of data that are needed to proceed. + /// Try to build the next `ParquetRecordBatchReader` for the active row group. /// - /// If a [`ParquetRecordBatchReader`] is ready, it is returned in - /// `DecodeResult::Data`. - pub(crate) fn try_build( - &mut self, - ) -> Result, ParquetError> { + /// Returns [`RowGroupBuildResult::NeedsData`] if more data is needed, + /// [`RowGroupBuildResult::Data`] if a reader is ready, or + /// [`RowGroupBuildResult::Finished`] if the row group completed without + /// producing a reader. + pub(crate) fn try_build(&mut self) -> Result { loop { let current_state = self.take_state()?; // Try to transition the decoder. @@ -310,18 +459,10 @@ impl RowGroupReaderBuilder { ) -> Result { let result = match current_state { RowGroupDecoderState::Start { row_group_info } => { - // Short-circuit once the overall output limit is exhausted. - // - // `self.limit` tracks how many more rows the reader is still - // allowed to emit and is decremented as each row group is - // planned in `StartData`, so `Some(0)` means earlier row - // groups have already produced the full requested output. - if matches!(self.limit, Some(0)) { - return Ok(NextState::result( - RowGroupDecoderState::Finished, - DecodeResult::Finished, - )); - } + debug_assert!( + !row_group_info.budget.is_exhausted(), + "RowGroupFrontier should not hand off row groups after the output limit is exhausted" + ); let column_chunks = None; // no prior column chunks @@ -371,6 +512,7 @@ impl RowGroupReaderBuilder { row_group_idx, row_count, plan_builder, + budget, } = row_group_info; // If nothing is selected, we are done with this row group @@ -379,7 +521,9 @@ impl RowGroupReaderBuilder { self.filter = Some(filter_info.into_filter()); return Ok(NextState::result( RowGroupDecoderState::Finished, - DecodeResult::Finished, + RowGroupBuildResult::Finished { + remaining_budget: budget, + }, )); } @@ -405,6 +549,7 @@ impl RowGroupReaderBuilder { row_group_idx, row_count, plan_builder, + budget, }; NextState::again(RowGroupDecoderState::WaitingOnFilterData { @@ -428,7 +573,7 @@ impl RowGroupReaderBuilder { filter_info, data_request, }, - DecodeResult::NeedsData(needed_ranges), + RowGroupBuildResult::NeedsData(needed_ranges), )); } @@ -437,6 +582,7 @@ impl RowGroupReaderBuilder { row_group_idx, row_count, mut plan_builder, + budget, } = row_group_info; let predicate = filter_info.current(); @@ -476,10 +622,10 @@ impl RowGroupReaderBuilder { // When this is the final predicate in the chain and an output // limit is set, tell the filter evaluation to stop once enough // matching rows have been accumulated. - let predicate_limit = self - .limit - .filter(|_| filter_info.is_last()) - .map(|l| l.saturating_add(self.offset.unwrap_or(0))); + let predicate_limit = filter_info + .is_last() + .then(|| budget.selected_row_limit()) + .flatten(); // Evaluate the filter via `with_predicate_options`, opting into // early termination when this is the final predicate and an @@ -495,6 +641,7 @@ impl RowGroupReaderBuilder { row_group_idx, row_count, plan_builder, + budget, }; // Take back the column chunks that were read @@ -531,47 +678,32 @@ impl RowGroupReaderBuilder { row_group_idx, row_count, plan_builder, + budget, } = row_group_info; - // Compute the number of rows in the selection before applying limit and offset - let rows_before = plan_builder.num_rows_selected().unwrap_or(row_count); + let BudgetedReadPlan { + mut plan_builder, + rows_before_budget, + rows_after_budget, + remaining_budget, + } = budget.apply_to_plan(plan_builder, row_count); - if rows_before == 0 { + if rows_before_budget == 0 { // ruled out entire row group return Ok(NextState::result( RowGroupDecoderState::Finished, - DecodeResult::Finished, + RowGroupBuildResult::Finished { remaining_budget }, )); } - // Apply any limit and offset - let mut plan_builder = plan_builder - .limited(row_count) - .with_offset(self.offset) - .with_limit(self.limit) - .build_limited(); - - let rows_after = plan_builder.num_rows_selected().unwrap_or(row_count); - - // Update running offset and limit for after the current row group is read - if let Some(offset) = &mut self.offset { - // Reduction is either because of offset or limit, as limit is applied - // after offset has been "exhausted" can just use saturating sub here - *offset = offset.saturating_sub(rows_before - rows_after) - } - - if rows_after == 0 { + if rows_after_budget == 0 { // no rows left after applying limit/offset return Ok(NextState::result( RowGroupDecoderState::Finished, - DecodeResult::Finished, + RowGroupBuildResult::Finished { remaining_budget }, )); } - if let Some(limit) = &mut self.limit { - *limit -= rows_after; - } - let data_request = DataRequestBuilder::new( row_group_idx, row_count, @@ -597,6 +729,7 @@ impl RowGroupReaderBuilder { row_group_idx, row_count, plan_builder, + budget: remaining_budget, }; NextState::again(RowGroupDecoderState::WaitingOnData { @@ -620,7 +753,7 @@ impl RowGroupReaderBuilder { data_request, cache_info, }, - DecodeResult::NeedsData(needed_ranges), + RowGroupBuildResult::NeedsData(needed_ranges), )); } @@ -629,6 +762,7 @@ impl RowGroupReaderBuilder { row_group_idx, row_count, plan_builder, + budget, } = row_group_info; let row_group = data_request.try_into_in_memory_row_group( @@ -656,11 +790,18 @@ impl RowGroupReaderBuilder { }?; let reader = ParquetRecordBatchReader::new(array_reader, plan); - NextState::result(RowGroupDecoderState::Finished, DecodeResult::Data(reader)) + NextState::result( + RowGroupDecoderState::Finished, + RowGroupBuildResult::Data { + batch_reader: reader, + remaining_budget: budget, + }, + ) } RowGroupDecoderState::Finished => { - // nothing left to read - NextState::result(RowGroupDecoderState::Finished, DecodeResult::Finished) + return Err(ParquetError::General(String::from( + "Internal Error: try_build called without an active row group", + ))); } }; Ok(result) @@ -760,10 +901,55 @@ fn override_selector_strategy_if_needed( #[cfg(test)] mod tests { use super::*; + use crate::arrow::arrow_reader::{RowSelection, RowSelector}; #[test] // Verify that the size of RowGroupDecoderState does not grow too large fn test_structure_size() { - assert_eq!(std::mem::size_of::(), 200); + assert_eq!(std::mem::size_of::(), 232); + } + + #[test] + fn test_row_budget_offset_limit_across_row_groups() { + let first = + RowBudget::new(Some(225), Some(20)).apply_to_plan(ReadPlanBuilder::new(1024), 200); + assert_eq!(first.rows_before_budget, 200); + assert_eq!(first.rows_after_budget, 0); + assert_eq!(first.remaining_budget, RowBudget::new(Some(25), Some(20))); + assert_eq!(first.plan_builder.num_rows_selected(), Some(0)); + + let second = first + .remaining_budget + .apply_to_plan(ReadPlanBuilder::new(1024), 200); + assert_eq!(second.rows_before_budget, 200); + assert_eq!(second.rows_after_budget, 20); + assert_eq!(second.remaining_budget, RowBudget::new(Some(0), Some(0))); + assert_eq!(second.plan_builder.num_rows_selected(), Some(20)); + } + + #[test] + fn test_row_budget_limit_only() { + let budgeted = + RowBudget::new(None, Some(20)).apply_to_plan(ReadPlanBuilder::new(1024), 200); + assert_eq!(budgeted.rows_before_budget, 200); + assert_eq!(budgeted.rows_after_budget, 20); + assert_eq!(budgeted.remaining_budget, RowBudget::new(None, Some(0))); + assert_eq!(budgeted.plan_builder.num_rows_selected(), Some(20)); + } + + #[test] + fn test_row_budget_empty_selection() { + let empty_selection = RowSelection::from(vec![RowSelector::skip(200)]); + let budgeted = RowBudget::new(Some(10), Some(20)).apply_to_plan( + ReadPlanBuilder::new(1024).with_selection(Some(empty_selection)), + 200, + ); + assert_eq!(budgeted.rows_before_budget, 0); + assert_eq!(budgeted.rows_after_budget, 0); + assert_eq!( + budgeted.remaining_budget, + RowBudget::new(Some(10), Some(20)) + ); + assert_eq!(budgeted.plan_builder.num_rows_selected(), Some(0)); } } diff --git a/parquet/src/arrow/push_decoder/remaining.rs b/parquet/src/arrow/push_decoder/remaining.rs index 2986ca0da8d8..aef105beb356 100644 --- a/parquet/src/arrow/push_decoder/remaining.rs +++ b/parquet/src/arrow/push_decoder/remaining.rs @@ -16,8 +16,13 @@ // under the License. use crate::DecodeResult; -use crate::arrow::arrow_reader::{ParquetRecordBatchReader, RowSelection}; -use crate::arrow::push_decoder::reader_builder::RowGroupReaderBuilder; +use crate::arrow::ProjectionMask; +use crate::arrow::arrow_reader::{ + ParquetRecordBatchReader, RowFilter, RowSelection, RowSelectionPolicy, +}; +use crate::arrow::push_decoder::reader_builder::{ + RowBudget, RowGroupBuildResult, RowGroupReaderBuilder, +}; use crate::errors::ParquetError; use crate::file::metadata::ParquetMetaData; use bytes::Bytes; @@ -25,21 +30,166 @@ use std::collections::VecDeque; use std::ops::Range; use std::sync::Arc; -/// State machine that tracks the remaining high level chunks (row groups) of -/// Parquet data are left to read. -/// -/// This is currently a row group, but the author aspires to extend the pattern -/// to data boundaries other than RowGroups in the future. +/// Plan for the next queued row group after row-selection slicing. #[derive(Debug)] -pub(crate) struct RemainingRowGroups { - /// The underlying Parquet metadata - parquet_metadata: Arc, +enum QueuedRowGroupDecision { + /// Hand this row group to the builder. + Read(NextRowGroup), + /// Skip this row group, and keep scanning with the updated budget. + Skip { remaining_budget: RowBudget }, +} - /// The row groups that have not yet been read - row_groups: VecDeque, +/// Work item handed from [`RowGroupFrontier`] to [`RowGroupReaderBuilder`]. +#[derive(Debug)] +struct NextRowGroup { + row_group_idx: usize, + row_count: usize, + /// This row group's slice of the global selection, or `None` when all rows + /// are selected. + selection: Option, + /// Budget snapshot to apply while decoding this row group. + budget: RowBudget, +} - /// Remaining selection to apply to the next row groups +#[derive(Debug)] +struct RowGroupFrontier { + /// Metadata used to resolve row counts for queued row groups. + parquet_metadata: Arc, + /// Row group indices not yet handed to the builder. + row_groups: VecDeque, + /// Cross-row-group cursor for the optional global row selection. selection: Option, + /// Offset/limit budget before the next readable row group is planned. + budget: RowBudget, + /// If predicates are present, row groups with selected rows must be read so + /// the predicate can decide whether they are actually needed. + has_predicates: bool, +} + +impl RowGroupFrontier { + fn new( + parquet_metadata: Arc, + row_groups: Vec, + selection: Option, + budget: RowBudget, + has_predicates: bool, + ) -> Self { + Self { + parquet_metadata, + row_groups: VecDeque::from(row_groups), + selection, + budget, + has_predicates, + } + } + + fn row_group_num_rows(&self, row_group_idx: usize) -> Result { + self.parquet_metadata + .row_group(row_group_idx) + .num_rows() + .try_into() + .map_err(|e| ParquetError::General(format!("Row count overflow: {e}"))) + } + + fn update_budget_after_row_group(&mut self, budget: RowBudget) { + self.budget = budget; + } + + fn clear_remaining(&mut self) { + self.selection = None; + self.row_groups.clear(); + } + + /// Plan whether a selected row group should be read or skipped. + /// + /// Selection-only skips are handled before this method is called. This + /// method applies the remaining offset/limit budget and predicate + /// conservatism. + fn plan_selected_row_group( + &self, + next_row_group: NextRowGroup, + selected_rows: usize, + ) -> QueuedRowGroupDecision { + if self.has_predicates { + return QueuedRowGroupDecision::Read(next_row_group); + } + + let rows_after_budget = self.budget.rows_after(selected_rows); + if rows_after_budget != 0 { + return QueuedRowGroupDecision::Read(next_row_group); + } + + QueuedRowGroupDecision::Skip { + remaining_budget: self.budget.advance(selected_rows, rows_after_budget), + } + } + + /// Advance queued row groups until one should be handed to the builder. + fn next_readable_row_group(&mut self) -> Result, ParquetError> { + loop { + let Some(&row_group_idx) = self.row_groups.front() else { + return Ok(None); + }; + if self.budget.is_exhausted() + || self + .selection + .as_ref() + .is_some_and(|selection| selection.row_count() == 0) + { + self.clear_remaining(); + return Ok(None); + } + + let row_count = self.row_group_num_rows(row_group_idx)?; + let (selection, selected_rows) = match self.selection.as_mut() { + Some(selection) => { + let selection = selection.split_off(row_count); + let selected_rows = selection.row_count(); + if selected_rows == 0 { + self.row_groups.pop_front(); + continue; + } + + let selection = if selected_rows == row_count { + None + } else { + Some(selection) + }; + (selection, selected_rows) + } + None => (None, row_count), + }; + + let next_row_group = NextRowGroup { + row_group_idx, + row_count, + selection, + budget: self.budget, + }; + + match self.plan_selected_row_group(next_row_group, selected_rows) { + QueuedRowGroupDecision::Read(next_row_group) => { + self.row_groups.pop_front(); + return Ok(Some(next_row_group)); + } + QueuedRowGroupDecision::Skip { remaining_budget } => { + self.row_groups.pop_front(); + self.budget = remaining_budget; + } + } + } + } +} + +/// State machine that tracks the remaining high level chunks (row groups) of +/// Parquet data left to read. +/// +/// [`RowGroupFrontier`] owns cross-row-group scan state and selects the next +/// work item. [`RowGroupReaderBuilder`] owns decoding for the active row group. +#[derive(Debug)] +pub(crate) struct RemainingRowGroups { + /// Cross-row-group scan state for queued work. + frontier: RowGroupFrontier, /// State for building the reader for the current row group row_group_reader_builder: RowGroupReaderBuilder, @@ -50,12 +200,18 @@ impl RemainingRowGroups { parquet_metadata: Arc, row_groups: Vec, selection: Option, + budget: RowBudget, + has_predicates: bool, row_group_reader_builder: RowGroupReaderBuilder, ) -> Self { Self { - parquet_metadata, - row_groups: VecDeque::from(row_groups), - selection, + frontier: RowGroupFrontier::new( + parquet_metadata, + row_groups, + selection, + budget, + has_predicates, + ), row_group_reader_builder, } } @@ -75,6 +231,40 @@ impl RemainingRowGroups { self.row_group_reader_builder.clear_all_ranges(); } + /// True iff the inner row-group reader is between row groups (state + /// `Finished`). Forward to [`RowGroupReaderBuilder::is_finished`]. + pub fn is_at_row_group_boundary(&self) -> bool { + self.row_group_reader_builder.is_finished() + } + + /// Number of row groups remaining (not including the one currently + /// being decoded). + pub fn row_groups_remaining(&self) -> usize { + self.frontier.row_groups.len() + } + + /// Replace the projection. Must be called when + /// [`Self::is_at_row_group_boundary`]. + pub fn set_projection(&mut self, projection: ProjectionMask) -> Result<(), ParquetError> { + self.row_group_reader_builder.set_projection(projection) + } + + /// Replace the row filter. Must be called when + /// [`Self::is_at_row_group_boundary`]. + pub fn set_filter(&mut self, filter: Option) -> Result<(), ParquetError> { + self.row_group_reader_builder.set_filter(filter) + } + + /// Replace the row selection policy. Must be called when + /// [`Self::is_at_row_group_boundary`]. + pub fn set_row_selection_policy( + &mut self, + policy: RowSelectionPolicy, + ) -> Result<(), ParquetError> { + self.row_group_reader_builder + .set_row_selection_policy(policy) + } + /// returns [`ParquetRecordBatchReader`] suitable for reading the next /// group of rows from the Parquet data, or the list of data ranges still /// needed to proceed @@ -82,42 +272,48 @@ impl RemainingRowGroups { &mut self, ) -> Result, ParquetError> { loop { - // Are we ready yet to start reading? - let result: DecodeResult = - self.row_group_reader_builder.try_build()?; - match result { - DecodeResult::Finished => { + if !self.row_group_reader_builder.has_active_row_group() { + // We are done with the previous row group, seek to the next one + // from the frontier, if any. + + match self.frontier.next_readable_row_group()? { + Some(NextRowGroup { + row_group_idx, + row_count, + selection, + budget, + }) => { + self.row_group_reader_builder.next_row_group( + row_group_idx, + row_count, + selection, + budget, + )?; + } + None => return Ok(DecodeResult::Finished), + } + } + + match self.row_group_reader_builder.try_build()? { + RowGroupBuildResult::Finished { remaining_budget } => { + self.frontier + .update_budget_after_row_group(remaining_budget); // reader is done, proceed to the next row group - // fall through to the next row group - // This happens if the row group was completely filtered out } - DecodeResult::NeedsData(ranges) => { + RowGroupBuildResult::NeedsData(ranges) => { // need more data to proceed return Ok(DecodeResult::NeedsData(ranges)); } - DecodeResult::Data(batch_reader) => { + RowGroupBuildResult::Data { + batch_reader, + remaining_budget, + } => { + self.frontier + .update_budget_after_row_group(remaining_budget); // ready to read the row group return Ok(DecodeResult::Data(batch_reader)); } } - - // No current reader, proceed to the next row group if any - let row_group_idx = match self.row_groups.pop_front() { - None => return Ok(DecodeResult::Finished), - Some(idx) => idx, - }; - - let row_count: usize = self - .parquet_metadata - .row_group(row_group_idx) - .num_rows() - .try_into() - .map_err(|e| ParquetError::General(format!("Row count overflow: {e}")))?; - - let selection = self.selection.as_mut().map(|s| s.split_off(row_count)); - self.row_group_reader_builder - .next_row_group(row_group_idx, row_count, selection)?; - // the next iteration will try to build the reader for the new row group } } } diff --git a/parquet/src/basic.rs b/parquet/src/basic.rs index ba8ffc2e92c3..17e5b9c321bb 100644 --- a/parquet/src/basic.rs +++ b/parquet/src/basic.rs @@ -28,7 +28,7 @@ pub use crate::compression::{BrotliLevel, GzipLevel, ZstdLevel}; use crate::file::metadata::HeapSize; use crate::parquet_thrift::{ ElementType, FieldType, ReadThrift, ThriftCompactInputProtocol, ThriftCompactOutputProtocol, - WriteThrift, WriteThriftField, + WriteThrift, WriteThriftField, validate_list_type, }; use crate::{thrift_enum, thrift_struct, thrift_union_all_empty, write_thrift_field}; @@ -771,6 +771,8 @@ impl<'a, R: ThriftCompactInputProtocol<'a>> ReadThrift<'a, R> for EncodingMask { // This reads a Thrift `list` and turns it into a bitmask let list_ident = prot.read_list_begin()?; + // check for enum (encoded as I32) + validate_list_type(ElementType::I32, &list_ident)?; for _ in 0..list_ident.size { let val = Encoding::read_thrift(prot)?; mask |= 1 << val as i32; @@ -798,6 +800,33 @@ fn i32_to_encoding(val: i32) -> Encoding { // ---------------------------------------------------------------------- // Mirrors thrift enum `CompressionCodec` +thrift_enum!( +/// Supported compression algorithms. +/// +/// Codecs added in format version X.Y can be read by readers based on X.Y and later. +/// Codec support may vary between readers based on the format version and +/// libraries available at runtime. +/// +/// See [Compression.md] for a detailed specification of these algorithms. +/// +/// [Compression.md]: https://github.com/apache/parquet-format/blob/master/Compression.md +enum CompressionCodec { + UNCOMPRESSED = 0; + SNAPPY = 1; + GZIP = 2; + LZO = 3; + BROTLI = 4; // Added in 2.4 + LZ4 = 5; // DEPRECATED (Added in 2.4) + ZSTD = 6; // Added in 2.4 + LZ4_RAW = 7; // Added in 2.9 +} +); + +// NOTE: This enum likely belongs in file::properties now, but moving it there would be a +// breaking API change, that's probably not worth the pain. If a new codec is added to the +// Parquet specification, or any other breaking changes are made to this enum, this can be +// revisited. + /// Supported block compression algorithms. /// /// Block compression can yield non-trivial improvements to storage efficiency at the expense @@ -834,51 +863,33 @@ pub enum Compression { LZ4_RAW, } -impl<'a, R: ThriftCompactInputProtocol<'a>> ReadThrift<'a, R> for Compression { - fn read_thrift(prot: &mut R) -> Result { - let val = prot.read_i32()?; - Ok(match val { - 0 => Self::UNCOMPRESSED, - 1 => Self::SNAPPY, - 2 => Self::GZIP(Default::default()), - 3 => Self::LZO, - 4 => Self::BROTLI(Default::default()), - 5 => Self::LZ4, - 6 => Self::ZSTD(Default::default()), - 7 => Self::LZ4_RAW, - _ => return Err(general_err!("Unexpected CompressionCodec {}", val)), - }) - } -} - -// TODO(ets): explore replacing this with a thrift_enum!(ThriftCompression) for the serialization -// and then provide `From` impls to convert back and forth. This is necessary due to the addition -// of compression level to some variants. -impl WriteThrift for Compression { - const ELEMENT_TYPE: ElementType = ElementType::I32; - - fn write_thrift(&self, writer: &mut ThriftCompactOutputProtocol) -> Result<()> { - let id: i32 = match *self { - Self::UNCOMPRESSED => 0, - Self::SNAPPY => 1, - Self::GZIP(_) => 2, - Self::LZO => 3, - Self::BROTLI(_) => 4, - Self::LZ4 => 5, - Self::ZSTD(_) => 6, - Self::LZ4_RAW => 7, - }; - writer.write_i32(id) +impl From for Compression { + fn from(value: CompressionCodec) -> Self { + match value { + CompressionCodec::UNCOMPRESSED => Compression::UNCOMPRESSED, + CompressionCodec::SNAPPY => Compression::SNAPPY, + CompressionCodec::GZIP => Compression::GZIP(Default::default()), + CompressionCodec::LZO => Compression::LZO, + CompressionCodec::BROTLI => Compression::BROTLI(Default::default()), + CompressionCodec::LZ4 => Compression::LZ4, + CompressionCodec::ZSTD => Compression::ZSTD(Default::default()), + CompressionCodec::LZ4_RAW => Compression::LZ4_RAW, + } } } -write_thrift_field!(Compression, FieldType::I32); - -impl Compression { - /// Returns the codec type of this compression setting as a string, without the compression - /// level. - pub(crate) fn codec_to_string(self) -> String { - format!("{self:?}").split('(').next().unwrap().to_owned() +impl From for CompressionCodec { + fn from(value: Compression) -> Self { + match value { + Compression::UNCOMPRESSED => CompressionCodec::UNCOMPRESSED, + Compression::SNAPPY => CompressionCodec::SNAPPY, + Compression::GZIP(_) => CompressionCodec::GZIP, + Compression::LZO => CompressionCodec::LZO, + Compression::BROTLI(_) => CompressionCodec::BROTLI, + Compression::LZ4 => CompressionCodec::LZ4, + Compression::ZSTD(_) => CompressionCodec::ZSTD, + Compression::LZ4_RAW => CompressionCodec::LZ4_RAW, + } } } @@ -2140,11 +2151,65 @@ mod tests { } #[test] - fn test_compression_codec_to_string() { - assert_eq!(Compression::UNCOMPRESSED.codec_to_string(), "UNCOMPRESSED"); + fn test_compression_conversion() { + assert_eq!( + CompressionCodec::from(Compression::UNCOMPRESSED), + CompressionCodec::UNCOMPRESSED + ); + assert_eq!( + CompressionCodec::from(Compression::SNAPPY), + CompressionCodec::SNAPPY + ); + assert_eq!( + CompressionCodec::from(Compression::GZIP(Default::default())), + CompressionCodec::GZIP + ); + assert_eq!( + CompressionCodec::from(Compression::LZO), + CompressionCodec::LZO + ); + assert_eq!( + CompressionCodec::from(Compression::BROTLI(Default::default())), + CompressionCodec::BROTLI + ); + assert_eq!( + CompressionCodec::from(Compression::LZ4), + CompressionCodec::LZ4 + ); + assert_eq!( + CompressionCodec::from(Compression::ZSTD(Default::default())), + CompressionCodec::ZSTD + ); + assert_eq!( + CompressionCodec::from(Compression::LZ4_RAW), + CompressionCodec::LZ4_RAW + ); + + assert_eq!( + Compression::from(CompressionCodec::UNCOMPRESSED), + Compression::UNCOMPRESSED + ); + assert_eq!( + Compression::from(CompressionCodec::SNAPPY), + Compression::SNAPPY + ); + assert_eq!( + Compression::from(CompressionCodec::GZIP), + Compression::GZIP(Default::default()) + ); + assert_eq!(Compression::from(CompressionCodec::LZO), Compression::LZO); + assert_eq!( + Compression::from(CompressionCodec::BROTLI), + Compression::BROTLI(Default::default()) + ); + assert_eq!(Compression::from(CompressionCodec::LZ4), Compression::LZ4); + assert_eq!( + Compression::from(CompressionCodec::ZSTD), + Compression::ZSTD(Default::default()) + ); assert_eq!( - Compression::ZSTD(ZstdLevel::default()).codec_to_string(), - "ZSTD" + Compression::from(CompressionCodec::LZ4_RAW), + Compression::LZ4_RAW ); } diff --git a/parquet/src/bin/parquet-layout.rs b/parquet/src/bin/parquet-layout.rs index 007f93517d96..735a9665139a 100644 --- a/parquet/src/bin/parquet-layout.rs +++ b/parquet/src/bin/parquet-layout.rs @@ -48,7 +48,7 @@ use parquet::file::metadata::ParquetMetaDataReader; use serde::Serialize; use thrift::protocol::TCompactInputProtocol; -use parquet::basic::Compression; +use parquet::basic::CompressionCodec; use parquet::errors::Result; use parquet::file::reader::ChunkReader; #[allow(deprecated)] @@ -118,7 +118,7 @@ fn do_layout(reader: &C) -> Result { .iter() .zip(schema.columns()) .map(|(column, column_schema)| { - let compression = compression(column.compression()); + let compression = compression(column.compression_codec()); let mut pages = vec![]; let mut start = column @@ -225,16 +225,16 @@ fn read_page_header(reader: &C, offset: u64) -> Result<(usize, P } /// Returns a string representation for a given compression -fn compression(compression: Compression) -> Option<&'static str> { +fn compression(compression: CompressionCodec) -> Option<&'static str> { match compression { - Compression::UNCOMPRESSED => None, - Compression::SNAPPY => Some("snappy"), - Compression::GZIP(_) => Some("gzip"), - Compression::LZO => Some("lzo"), - Compression::BROTLI(_) => Some("brotli"), - Compression::LZ4 => Some("lz4"), - Compression::ZSTD(_) => Some("zstd"), - Compression::LZ4_RAW => Some("lz4_raw"), + CompressionCodec::UNCOMPRESSED => None, + CompressionCodec::SNAPPY => Some("snappy"), + CompressionCodec::GZIP => Some("gzip"), + CompressionCodec::LZO => Some("lzo"), + CompressionCodec::BROTLI => Some("brotli"), + CompressionCodec::LZ4 => Some("lz4"), + CompressionCodec::ZSTD => Some("zstd"), + CompressionCodec::LZ4_RAW => Some("lz4_raw"), } } diff --git a/parquet/src/bin/parquet-rewrite.rs b/parquet/src/bin/parquet-rewrite.rs index 5bdd432be530..2428c491413e 100644 --- a/parquet/src/bin/parquet-rewrite.rs +++ b/parquet/src/bin/parquet-rewrite.rs @@ -279,6 +279,10 @@ struct Args { #[clap(long)] write_page_header_statistics: Option, + /// Write path_in_schema to the column metadata. + #[clap(long)] + write_path_in_schema: Option, + /// Sets whether bloom filter is enabled for all columns. #[clap(long)] bloom_filter_enabled: Option, @@ -380,7 +384,8 @@ fn main() { writer_properties_builder = writer_properties_builder.set_bloom_filter_fpp(value); } if let Some(value) = args.bloom_filter_ndv { - writer_properties_builder = writer_properties_builder.set_bloom_filter_ndv(value); + writer_properties_builder = + writer_properties_builder.set_bloom_filter_max_ndv(value); } if let Some(value) = args.bloom_filter_position { writer_properties_builder = @@ -406,6 +411,9 @@ fn main() { if let Some(value) = args.coerce_types { writer_properties_builder = writer_properties_builder.set_coerce_types(value); } + if let Some(value) = args.write_path_in_schema { + writer_properties_builder = writer_properties_builder.set_write_path_in_schema(value); + } if let Some(value) = args.write_batch_size { writer_properties_builder = writer_properties_builder.set_write_batch_size(value); } diff --git a/parquet/src/column/chunker/cdc.rs b/parquet/src/column/chunker/cdc.rs index cb4328536df3..b40dd74a8d83 100644 --- a/parquet/src/column/chunker/cdc.rs +++ b/parquet/src/column/chunker/cdc.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "arrow")] +use crate::column::writer::LevelDataRef; use crate::errors::{ParquetError, Result}; use crate::file::properties::CdcOptions; use crate::schema::types::ColumnDescriptor; @@ -275,8 +277,8 @@ impl ContentDefinedChunker { /// evaluate if we need to create a new chunk. fn calculate( &mut self, - def_levels: Option<&[i16]>, - rep_levels: Option<&[i16]>, + def_levels: LevelDataRef<'_>, + rep_levels: LevelDataRef<'_>, num_levels: usize, mut roll_value: F, ) -> Vec @@ -322,10 +324,11 @@ impl ContentDefinedChunker { // def_levels: [1, 0, 1, 0, 1] // level: 0 1 2 3 4 // value_offset: 0 1 2 (only increments on def==1) - let def_levels = def_levels.expect("def_levels required when max_def_level > 0"); #[allow(clippy::needless_range_loop)] for offset in 0..num_levels { - let def_level = def_levels[offset]; + let def_level = def_levels + .value_at(offset) + .expect("def_levels required when max_def_level > 0"); self.roll_level(def_level); if def_level == self.max_def_level { // For non-nested data, the leaf array has one slot per @@ -380,13 +383,15 @@ impl ContentDefinedChunker { // Using value_offset=1 would index position 1 (the null slot). // // Using value_offset for roll_value would hash the wrong array slot. - let def_levels = def_levels.expect("def_levels required for nested data"); - let rep_levels = rep_levels.expect("rep_levels required for nested data"); let mut leaf_offset: usize = 0; for offset in 0..num_levels { - let def_level = def_levels[offset]; - let rep_level = rep_levels[offset]; + let def_level = def_levels + .value_at(offset) + .expect("def_levels required for nested data"); + let rep_level = rep_levels + .value_at(offset) + .expect("rep_levels required for nested data"); self.roll_level(def_level); self.roll_level(rep_level); @@ -439,16 +444,20 @@ impl ContentDefinedChunker { #[cfg(feature = "arrow")] pub(crate) fn get_arrow_chunks( &mut self, - def_levels: Option<&[i16]>, - rep_levels: Option<&[i16]>, + def_levels: LevelDataRef<'_>, + rep_levels: LevelDataRef<'_>, array: &dyn arrow_array::Array, ) -> Result> { use arrow_array::cast::AsArray; use arrow_schema::DataType; - let num_levels = match def_levels { - Some(def_levels) => def_levels.len(), - None => array.len(), + // For nested (list) data, null list entries can own non-zero child + // ranges in the leaf array, so `array.len()` may exceed the number of + // levels. Always drive the loop by the level count; fall back to the + // array length only when there are no levels at all. + let num_levels = match (def_levels.len(), rep_levels.len()) { + (0, 0) => array.len(), + (d, r) => d.max(r), }; macro_rules! fixed_width { @@ -566,6 +575,8 @@ impl ContentDefinedChunker { mod tests { use super::*; use crate::basic::Type as PhysicalType; + #[cfg(feature = "arrow")] + use crate::column::writer::LevelDataRef; use crate::schema::types::{ColumnPath, Type}; use std::sync::Arc; @@ -615,9 +626,14 @@ mod tests { // Write a small amount of data — should produce exactly 1 chunk. let num_values = 4; - let chunks = chunker.calculate(None, None, num_values, |c, i| { - c.roll_fixed::<4>(&(i as i32).to_le_bytes()); - }); + let chunks = chunker.calculate( + LevelDataRef::Absent, + LevelDataRef::Absent, + num_values, + |c, i| { + c.roll_fixed::<4>(&(i as i32).to_le_bytes()); + }, + ); assert_eq!(chunks.len(), 1); assert_eq!(chunks[0].level_offset, 0); assert_eq!(chunks[0].value_offset, 0); @@ -636,9 +652,14 @@ mod tests { // Write enough data to exceed max_chunk_size multiple times. // Each i32 = 4 bytes, max_chunk_size=1024, so ~256 values per chunk max. let num_values = 2000; - let chunks = chunker.calculate(None, None, num_values, |c, i| { - c.roll_fixed::<4>(&(i as i32).to_le_bytes()); - }); + let chunks = chunker.calculate( + LevelDataRef::Absent, + LevelDataRef::Absent, + num_values, + |c, i| { + c.roll_fixed::<4>(&(i as i32).to_le_bytes()); + }, + ); // Should have multiple chunks assert!(chunks.len() > 1); @@ -668,10 +689,10 @@ mod tests { }; let mut chunker1 = ContentDefinedChunker::new(&make_desc(0, 0), &options).unwrap(); - let chunks1 = chunker1.calculate(None, None, 200, roll); + let chunks1 = chunker1.calculate(LevelDataRef::Absent, LevelDataRef::Absent, 200, roll); let mut chunker2 = ContentDefinedChunker::new(&make_desc(0, 0), &options).unwrap(); - let chunks2 = chunker2.calculate(None, None, 200, roll); + let chunks2 = chunker2.calculate(LevelDataRef::Absent, LevelDataRef::Absent, 200, roll); assert_eq!(chunks1.len(), chunks2.len()); for (a, b) in chunks1.iter().zip(chunks2.iter()) { @@ -699,9 +720,14 @@ mod tests { .collect(); let expected_non_null: usize = def_levels.iter().filter(|&&d| d == 1).count(); - let chunks = chunker.calculate(Some(&def_levels), None, num_levels, |c, i| { - c.roll_fixed::<4>(&(i as i32).to_le_bytes()); - }); + let chunks = chunker.calculate( + LevelDataRef::Materialized(&def_levels), + LevelDataRef::Absent, + num_levels, + |c, i| { + c.roll_fixed::<4>(&(i as i32).to_le_bytes()); + }, + ); assert!(!chunks.is_empty()); let total_levels: usize = chunks.iter().map(|c| c.num_levels).sum(); @@ -729,6 +755,7 @@ mod arrow_tests { use crate::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use crate::arrow::arrow_writer::ArrowWriter; + use crate::column::writer::LevelDataRef; use crate::file::properties::{CdcOptions, WriterProperties}; use crate::file::reader::{FileReader, SerializedFileReader}; @@ -1109,6 +1136,27 @@ mod arrow_tests { buf } + #[test] + fn cdc_all_null_arrow_column_writes_data_pages() { + let array = Arc::new(Int32Array::from(vec![None::; 4096])) as ArrayRef; + let schema = Arc::new(Schema::new(vec![Field::new("f0", DataType::Int32, true)])); + let batch = RecordBatch::try_new(schema, vec![array.clone()]).unwrap(); + + let data = write_with_cdc_options(&[&batch], 64, 256, Some(4096), false); + let info = get_column_info(&data, 0); + + assert_eq!(info.len(), 1); + assert!( + !info[0].page_lengths.is_empty(), + "all-null CDC write should still emit data pages" + ); + assert_eq!( + info[0].page_lengths.iter().sum::(), + array.len() as i64, + "all-null CDC pages should account for every input row" + ); + } + fn read_batches(data: &[u8]) -> Vec { let reader = ParquetRecordBatchReaderBuilder::try_new(bytes::Bytes::from(data.to_vec())) .unwrap() @@ -2190,11 +2238,15 @@ mod arrow_tests { let array: Int32Array = (0..n).map(|i| test_hash(0, i as u64) as i32).collect(); let mut chunker = super::ContentDefinedChunker::new(&desc, &options).unwrap(); - let chunks = chunker.get_arrow_chunks(None, None, &array).unwrap(); + let chunks = chunker + .get_arrow_chunks(LevelDataRef::Absent, LevelDataRef::Absent, &array) + .unwrap(); let sliced = array.slice(offset, n - offset); let mut chunker2 = super::ContentDefinedChunker::new(&desc, &options).unwrap(); - let chunks2 = chunker2.get_arrow_chunks(None, None, &sliced).unwrap(); + let chunks2 = chunker2 + .get_arrow_chunks(LevelDataRef::Absent, LevelDataRef::Absent, &sliced) + .unwrap(); let values: Vec = chunks.iter().map(|c| c.num_values).collect(); let values2: Vec = chunks2.iter().map(|c| c.num_values).collect(); diff --git a/parquet/src/column/writer/mod.rs b/parquet/src/column/writer/mod.rs index f755beed55c4..5d14ac6856f9 100644 --- a/parquet/src/column/writer/mod.rs +++ b/parquet/src/column/writer/mod.rs @@ -314,6 +314,67 @@ impl ColumnMetrics { } } +/// Borrowed view of level data, analogous to `&str` for `LevelData`'s `String`. +/// +/// `LevelDataRef` can be constructed from `LevelData` and directly from an existing +/// `&[i16]` without allocating. +/// +/// The variants are different physical representations of the same logical +/// sequence of levels. +#[derive(Debug, Clone, Copy)] +pub(crate) enum LevelDataRef<'a> { + Absent, + Materialized(&'a [i16]), + Uniform { value: i16, count: usize }, +} + +impl<'a> From<&'a [i16]> for LevelDataRef<'a> { + fn from(levels: &'a [i16]) -> Self { + Self::Materialized(levels) + } +} + +impl<'a> From> for LevelDataRef<'a> { + fn from(levels: Option<&'a [i16]>) -> Self { + levels.map_or(Self::Absent, Self::from) + } +} + +impl<'a> LevelDataRef<'a> { + pub(crate) fn len(self) -> usize { + match self { + Self::Absent => 0, + Self::Materialized(values) => values.len(), + Self::Uniform { count, .. } => count, + } + } + + pub(crate) fn first(self) -> Option { + match self { + Self::Absent => None, + Self::Materialized(values) => values.first().copied(), + Self::Uniform { value, count } => (count > 0).then_some(value), + } + } + + #[cfg(feature = "arrow")] + pub(crate) fn value_at(self, idx: usize) -> Option { + match self { + Self::Absent => None, + Self::Materialized(values) => values.get(idx).copied(), + Self::Uniform { value, count } => (idx < count).then_some(value), + } + } + + pub(crate) fn slice(self, offset: usize, len: usize) -> Self { + match self { + Self::Absent => Self::Absent, + Self::Materialized(values) => Self::Materialized(&values[offset..offset + len]), + Self::Uniform { value, .. } => Self::Uniform { value, count: len }, + } + } +} + /// Typed column writer for a primitive column. pub type ColumnWriterImpl<'a, T> = GenericColumnWriter<'a, ColumnValueEncoderImpl>; @@ -424,21 +485,19 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { &mut self, values: &E::Values, value_indices: Option<&[usize]>, - def_levels: Option<&[i16]>, - rep_levels: Option<&[i16]>, + def_levels: LevelDataRef<'_>, + rep_levels: LevelDataRef<'_>, min: Option<&E::T>, max: Option<&E::T>, distinct_count: Option, ) -> Result { // Check if number of definition levels is the same as number of repetition levels. - if let (Some(def), Some(rep)) = (def_levels, rep_levels) { - if def.len() != rep.len() { - return Err(general_err!( - "Inconsistent length of definition and repetition levels: {} != {}", - def.len(), - rep.len() - )); - } + if def_levels.len() != 0 && rep_levels.len() != 0 && def_levels.len() != rep_levels.len() { + return Err(general_err!( + "Inconsistent length of definition and repetition levels: {} != {}", + def_levels.len(), + rep_levels.len() + )); } // We check for DataPage limits only after we have inserted the values. If a user @@ -450,10 +509,11 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { // TODO: find out why we don't account for size of levels when we estimate page // size. - - let num_levels = match def_levels { - Some(def_levels) => def_levels.len(), - None => values.len(), + let num_levels = def_levels.len().max(rep_levels.len()); + let num_levels = if num_levels > 0 { + num_levels + } else { + value_indices.map_or(values.len(), |i| i.len()) }; if let Some(min) = min { @@ -472,13 +532,24 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { let mut values_offset = 0; let mut levels_offset = 0; - let base_batch_size = self.props.write_batch_size(); + let both_levels_compact = !matches!(def_levels, LevelDataRef::Materialized(_)) + && !matches!(rep_levels, LevelDataRef::Materialized(_)); + let has_levels = !matches!(def_levels, LevelDataRef::Absent) + || !matches!(rep_levels, LevelDataRef::Absent); + // When both level vectors are compact (Uniform or Absent), there is no + // materialized slice to split and the per-mini-batch work is O(1), so we + // can safely use a much larger batch size. + let base_batch_size = if both_levels_compact && has_levels { + self.props.data_page_row_count_limit() + } else { + self.props.write_batch_size() + }; while levels_offset < num_levels { let mut end_offset = num_levels.min(levels_offset + base_batch_size); // Split at record boundary - if let Some(r) = rep_levels { - while end_offset < r.len() && r[end_offset] != 0 { + if let LevelDataRef::Materialized(levels) = rep_levels { + while end_offset < levels.len() && levels[end_offset] != 0 { end_offset += 1; } } @@ -488,8 +559,8 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { values_offset, value_indices, end_offset - levels_offset, - def_levels.map(|lv| &lv[levels_offset..end_offset]), - rep_levels.map(|lv| &lv[levels_offset..end_offset]), + def_levels.slice(levels_offset, end_offset - levels_offset), + rep_levels.slice(levels_offset, end_offset - levels_offset), )?; levels_offset = end_offset; } @@ -516,7 +587,15 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { def_levels: Option<&[i16]>, rep_levels: Option<&[i16]>, ) -> Result { - self.write_batch_internal(values, None, def_levels, rep_levels, None, None, None) + self.write_batch_internal( + values, + None, + LevelDataRef::from(def_levels), + LevelDataRef::from(rep_levels), + None, + None, + None, + ) } /// Writer may optionally provide pre-calculated statistics for use when computing @@ -538,8 +617,8 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { self.write_batch_internal( values, None, - def_levels, - rep_levels, + LevelDataRef::from(def_levels), + LevelDataRef::from(rep_levels), min, max, distinct_count, @@ -650,32 +729,53 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { values_offset: usize, value_indices: Option<&[usize]>, num_levels: usize, - def_levels: Option<&[i16]>, - rep_levels: Option<&[i16]>, + def_levels: LevelDataRef<'_>, + rep_levels: LevelDataRef<'_>, ) -> Result { // Process definition levels and determine how many values to write. let values_to_write = if self.descr.max_def_level() > 0 { - let levels = def_levels.ok_or_else(|| { - general_err!( - "Definition levels are required, because max definition level = {}", - self.descr.max_def_level() - ) - })?; - - let mut values_to_write = 0usize; let max_def = self.descr.max_def_level(); - let encoder = &mut self.def_levels_encoder; - match self.page_metrics.definition_level_histogram.as_mut() { - Some(histogram) => encoder.put_with_observer(levels, |level, count| { - values_to_write += count * (level == max_def) as usize; - histogram.increment_by(level, count as i64); - }), - None => encoder.put_with_observer(levels, |level, count| { - values_to_write += count * (level == max_def) as usize; - }), - }; - self.page_metrics.num_page_nulls += (levels.len() - values_to_write) as u64; - values_to_write + match def_levels { + LevelDataRef::Absent => { + return Err(general_err!( + "Definition levels are required, because max definition level = {}", + self.descr.max_def_level() + )); + } + LevelDataRef::Materialized(levels) => { + // General path for caller-provided or already-materialized + // level buffers. + let mut values_to_write = 0usize; + let encoder = &mut self.def_levels_encoder; + match self.page_metrics.definition_level_histogram.as_mut() { + Some(histogram) => encoder.put_with_observer(levels, |level, count| { + values_to_write += count * (level == max_def) as usize; + histogram.increment_by(level, count as i64); + }), + None => encoder.put_with_observer(levels, |level, count| { + values_to_write += count * (level == max_def) as usize; + }), + }; + self.page_metrics.num_page_nulls += (levels.len() - values_to_write) as u64; + values_to_write + } + LevelDataRef::Uniform { value, count } => { + // Fast path for all-null, all-valid, or otherwise uniform + // definition levels without materializing a level buffer. + let encoder = &mut self.def_levels_encoder; + match self.page_metrics.definition_level_histogram.as_mut() { + Some(histogram) => { + encoder.put_n_with_observer(value, count, |level, run_len| { + histogram.increment_by(level, run_len as i64); + }) + } + None => encoder.put_n_with_observer(value, count, |_, _| {}), + }; + let values_to_write = count * (value == max_def) as usize; + self.page_metrics.num_page_nulls += (count - values_to_write) as u64; + values_to_write + } + } } else { num_levels }; @@ -683,31 +783,50 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { // Process repetition levels and determine how many rows we are about to process. if self.descr.max_rep_level() > 0 { // A row could contain more than one value. - let levels = rep_levels.ok_or_else(|| { + let first_level = rep_levels.first().ok_or_else(|| { general_err!( "Repetition levels are required, because max repetition level = {}", self.descr.max_rep_level() ) })?; - if !levels.is_empty() && levels[0] != 0 { + if first_level != 0 { return Err(general_err!( "Write must start at a record boundary, got non-zero repetition level of {}", - levels[0] + first_level )); } let mut new_rows = 0u32; - let encoder = &mut self.rep_levels_encoder; - match self.page_metrics.repetition_level_histogram.as_mut() { - Some(histogram) => encoder.put_with_observer(levels, |level, count| { - new_rows += (count as u32) * (level == 0) as u32; - histogram.increment_by(level, count as i64); - }), - None => encoder.put_with_observer(levels, |level, count| { - new_rows += (count as u32) * (level == 0) as u32; - }), - }; + match rep_levels { + LevelDataRef::Absent => unreachable!(), + LevelDataRef::Materialized(levels) => { + let encoder = &mut self.rep_levels_encoder; + match self.page_metrics.repetition_level_histogram.as_mut() { + Some(histogram) => encoder.put_with_observer(levels, |level, count| { + new_rows += (count as u32) * (level == 0) as u32; + histogram.increment_by(level, count as i64); + }), + None => encoder.put_with_observer(levels, |level, count| { + new_rows += (count as u32) * (level == 0) as u32; + }), + }; + } + LevelDataRef::Uniform { value, count } => { + let encoder = &mut self.rep_levels_encoder; + match self.page_metrics.repetition_level_histogram.as_mut() { + Some(histogram) => { + encoder.put_n_with_observer(value, count, |level, run_len| { + new_rows += (run_len as u32) * (level == 0) as u32; + histogram.increment_by(level, run_len as i64); + }) + } + None => encoder.put_n_with_observer(value, count, |level, run_len| { + new_rows += (run_len as u32) * (level == 0) as u32; + }), + }; + } + } self.page_metrics.num_buffered_rows += new_rows; } else { // Each value is exactly one row. @@ -4451,4 +4570,221 @@ mod tests { result.metadata.compressed_size() ); } + + struct ColumnRoundTripUniform<'a, T: DataType> { + props: WriterProperties, + values: &'a [T::T], + def_levels: LevelDataRef<'a>, + rep_levels: LevelDataRef<'a>, + max_def_level: i16, + max_rep_level: i16, + expected_values: &'a [T::T], + expected_def_levels: Option<&'a [i16]>, + expected_rep_levels: Option<&'a [i16]>, + } + + impl<'a, T: DataType> ColumnRoundTripUniform<'a, T> + where + T::T: PartialEq + std::fmt::Debug, + { + fn new() -> Self { + Self { + props: Default::default(), + values: &[], + def_levels: LevelDataRef::Absent, + rep_levels: LevelDataRef::Absent, + max_def_level: 0, + max_rep_level: 0, + expected_values: &[], + expected_def_levels: None, + expected_rep_levels: None, + } + } + + fn with_props(mut self, props: WriterProperties) -> Self { + self.props = props; + self + } + + fn with_values(mut self, values: &'a [T::T]) -> Self { + self.values = values; + self + } + + fn with_def_levels(mut self, def_levels: LevelDataRef<'a>) -> Self { + self.def_levels = def_levels; + self + } + + fn with_rep_levels(mut self, rep_levels: LevelDataRef<'a>) -> Self { + self.rep_levels = rep_levels; + self + } + + fn with_max_def_level(mut self, max_def_level: i16) -> Self { + self.max_def_level = max_def_level; + self + } + + fn with_max_rep_level(mut self, max_rep_level: i16) -> Self { + self.max_rep_level = max_rep_level; + self + } + + fn with_expected_values(mut self, expected_values: &'a [T::T]) -> Self { + self.expected_values = expected_values; + self + } + + fn with_expected_def_levels(mut self, expected_def_levels: &'a [i16]) -> Self { + self.expected_def_levels = Some(expected_def_levels); + self + } + + fn with_expected_rep_levels(mut self, expected_rep_levels: &'a [i16]) -> Self { + self.expected_rep_levels = Some(expected_rep_levels); + self + } + + /// Write-then-read roundtrip using `write_batch_internal` with the given + /// [`LevelDataRef`] variants, and assert the read-back matches `expected_*`. + fn run(self) { + let mut file = tempfile::tempfile().unwrap(); + let mut write = TrackedWrite::new(&mut file); + let page_writer = Box::new(SerializedPageWriter::new(&mut write)); + let mut writer = get_test_column_writer::( + page_writer, + self.max_def_level, + self.max_rep_level, + Arc::new(self.props), + ); + + writer + .write_batch_internal( + self.values, + None, + self.def_levels, + self.rep_levels, + None, + None, + None, + ) + .unwrap(); + let result = writer.close().unwrap(); + drop(write); + + let props = ReaderProperties::builder() + .set_backward_compatible_lz4(false) + .build(); + let page_reader = Box::new( + SerializedPageReader::new_with_properties( + Arc::new(file), + &result.metadata, + result.rows_written as usize, + None, + Arc::new(props), + ) + .unwrap(), + ); + let mut reader = + get_test_column_reader::(page_reader, self.max_def_level, self.max_rep_level); + + let batch_size = self + .expected_def_levels + .map_or(self.expected_values.len(), |l| l.len()); + let mut actual_values = Vec::with_capacity(batch_size); + let mut actual_def = self + .expected_def_levels + .map(|_| Vec::with_capacity(batch_size)); + let mut actual_rep = self + .expected_rep_levels + .map(|_| Vec::with_capacity(batch_size)); + + let (_, values_read, levels_read) = reader + .read_records( + batch_size, + actual_def.as_mut(), + actual_rep.as_mut(), + &mut actual_values, + ) + .unwrap(); + + assert_eq!(&actual_values[..values_read], self.expected_values); + if let Some(ref v) = actual_def { + assert_eq!(&v[..levels_read], self.expected_def_levels.unwrap()); + } + if let Some(ref v) = actual_rep { + assert_eq!(&v[..levels_read], self.expected_rep_levels.unwrap()); + } + } + } + + #[test] + fn test_uniform_def_levels_all_null() { + // All-null column: def_level=0 (null) for every slot, no values written. + let max_def_level = 1; + let count = 100; + let expected_def_levels = vec![0i16; count]; + ColumnRoundTripUniform::::new() + .with_def_levels(LevelDataRef::Uniform { value: 0, count }) + .with_max_def_level(max_def_level) + .with_expected_def_levels(&expected_def_levels) + .run(); + } + + #[test] + fn test_uniform_def_levels_all_valid() { + // All-valid column: def_level=max for every slot, all values written. + let max_def_level = 1; + let values: Vec = (0..50).collect(); + let expected_def_levels = vec![max_def_level; values.len()]; + ColumnRoundTripUniform::::new() + .with_values(&values) + .with_def_levels(LevelDataRef::Uniform { + value: max_def_level, + count: values.len(), + }) + .with_max_def_level(max_def_level) + .with_expected_values(&values) + .with_expected_def_levels(&expected_def_levels) + .run(); + } + + #[test] + fn test_uniform_def_and_rep_levels() { + // Simulates a list column where every row is null: + // def=0, rep=0 for each row (one row = one entry with no child values). + let max_def_level = 2; + let max_rep_level = 1; + let count = 200; + let expected_def_levels = vec![0i16; count]; + let expected_rep_levels = vec![0i16; count]; + ColumnRoundTripUniform::::new() + .with_def_levels(LevelDataRef::Uniform { value: 0, count }) + .with_rep_levels(LevelDataRef::Uniform { value: 0, count }) + .with_max_def_level(max_def_level) + .with_max_rep_level(max_rep_level) + .with_expected_def_levels(&expected_def_levels) + .with_expected_rep_levels(&expected_rep_levels) + .run(); + } + + #[test] + fn test_uniform_levels_v1_and_v2() { + // Verify uniform levels work identically for both Parquet writer versions. + for version in [WriterVersion::PARQUET_1_0, WriterVersion::PARQUET_2_0] { + let props = WriterProperties::builder() + .set_writer_version(version) + .build(); + let max_def = 1; + let count = 100; + let expected_def_levels = vec![0i16; count]; + ColumnRoundTripUniform::::new() + .with_props(props) + .with_def_levels(LevelDataRef::Uniform { value: 0, count }) + .with_max_def_level(max_def) + .with_expected_def_levels(&expected_def_levels) + .run(); + } + } } diff --git a/parquet/src/encodings/levels.rs b/parquet/src/encodings/levels.rs index 6b6de8b0e1c6..841afd8b3186 100644 --- a/parquet/src/encodings/levels.rs +++ b/parquet/src/encodings/levels.rs @@ -92,6 +92,37 @@ impl LevelEncoder { } } + /// Encode `count` repetitions of a single level value, calling + /// `observer(value, count)` exactly once. + /// + /// This can be used to encode uniform runs without allocating a level + /// buffer. Like [`Self::put_with_observer`], but specialized for a single + /// repeated level value. + /// + /// This is O(1) amortized for RLE-based encoders (after a small warmup). + #[inline] + pub fn put_n_with_observer(&mut self, value: i16, count: usize, mut observer: F) + where + F: FnMut(i16, usize), + { + let encoder = match *self { + LevelEncoder::Rle(ref mut encoder) | LevelEncoder::RleV2(ref mut encoder) => encoder, + }; + + // Feed values individually until the encoder enters RLE accumulation + // mode for this value, or until we've encoded everything. + let mut remaining = count; + while remaining > 0 && !encoder.is_accumulating_rle(value as u64) { + encoder.put(value as u64); + remaining -= 1; + } + // If we're now in accumulation mode, bulk-extend the rest. + if remaining > 0 { + encoder.extend_run(remaining); + } + observer(value, count); + } + /// Finalizes level encoder, flush all intermediate buffers and return resulting /// encoded buffer. Returned buffer is already truncated to encoded bytes only. #[inline] @@ -139,3 +170,116 @@ impl LevelEncoder { result } } + +#[cfg(test)] +mod tests { + use super::*; + + /// Encode `count` repetitions of `value` using `put_with_observer` and + /// return the raw encoded bytes. + fn reference_encode(max_level: i16, values: &[i16]) -> Vec { + let mut enc = LevelEncoder::v2_streaming(max_level); + enc.put_with_observer(values, |_, _| {}); + enc.consume() + } + + #[test] + fn test_put_n_with_observer_large_run() { + // Large count exercises the bulk extend_run path (past the 8-value warmup). + let max_level = 3; + let count = 10_000; + let mut enc = LevelEncoder::v2_streaming(max_level); + enc.put_n_with_observer(2, count, |_, _| {}); + assert_eq!(enc.consume(), reference_encode(max_level, &vec![2; count])); + } + + #[test] + fn test_put_n_with_observer_small_count() { + // Count smaller than the RLE warmup threshold — only the per-element loop runs. + let max_level = 3; + let mut enc = LevelEncoder::v2_streaming(max_level); + enc.put_n_with_observer(1, 5, |_, _| {}); + assert_eq!(enc.consume(), reference_encode(max_level, &[1; 5])); + } + + #[test] + fn test_put_n_with_observer_exact_threshold() { + // Exactly 8 values: the warmup loop completes and extend_run gets 0. + let max_level = 3; + let mut enc = LevelEncoder::v2_streaming(max_level); + enc.put_n_with_observer(3, 8, |_, _| {}); + assert_eq!(enc.consume(), reference_encode(max_level, &[3; 8])); + } + + #[test] + fn test_put_n_with_observer_single_value() { + let max_level = 1; + let mut enc = LevelEncoder::v2_streaming(max_level); + enc.put_n_with_observer(1, 1, |_, _| {}); + assert_eq!(enc.consume(), reference_encode(max_level, &[1])); + } + + #[test] + fn test_put_n_with_observer_zero_count() { + let max_level = 3; + let mut enc = LevelEncoder::v2_streaming(max_level); + enc.put_n_with_observer(2, 0, |_, _| {}); + assert_eq!(enc.consume(), reference_encode(max_level, &[])); + } + + #[test] + fn test_put_n_with_observer_calls_observer_exactly_once() { + let mut enc = LevelEncoder::v2_streaming(3); + let mut calls: Vec<(i16, usize)> = Vec::new(); + enc.put_n_with_observer(2, 500, |val, cnt| calls.push((val, cnt))); + assert_eq!(calls, vec![(2, 500)]); + } + + #[test] + fn test_put_n_with_observer_zero_count_calls_observer() { + let mut enc = LevelEncoder::v2_streaming(3); + let mut calls: Vec<(i16, usize)> = Vec::new(); + enc.put_n_with_observer(1, 0, |val, cnt| calls.push((val, cnt))); + assert_eq!(calls, vec![(1, 0)]); + } + + #[test] + fn test_put_n_with_observer_followed_by_different_value() { + // Two consecutive put_n calls with different values — verifies that + // the encoder correctly transitions between runs. + let max_level = 3; + let mut enc = LevelEncoder::v2_streaming(max_level); + let mut calls: Vec<(i16, usize)> = Vec::new(); + enc.put_n_with_observer(1, 100, |v, c| calls.push((v, c))); + enc.put_n_with_observer(3, 200, |v, c| calls.push((v, c))); + assert_eq!(calls, vec![(1, 100), (3, 200)]); + + let reference = reference_encode(max_level, &[&[1i16; 100][..], &[3i16; 200]].concat()); + assert_eq!(enc.consume(), reference); + } + + #[test] + fn test_put_n_with_observer_interleaved_with_put_with_observer() { + // Mix put_n_with_observer and put_with_observer to verify they compose. + let max_level = 3; + let mut enc = LevelEncoder::v2_streaming(max_level); + enc.put_n_with_observer(2, 50, |_, _| {}); + enc.put_with_observer(&[0, 0, 1, 1, 3], |_, _| {}); + enc.put_n_with_observer(2, 50, |_, _| {}); + + let input = [&[2i16; 50][..], &[0, 0, 1, 1, 3], &[2i16; 50]].concat(); + assert_eq!(enc.consume(), reference_encode(max_level, &input)); + } + + #[test] + fn test_put_n_with_observer_v1_roundtrip() { + // Also verify V1 (Rle variant with length header) works correctly. + let max_level = 3; + let mut enc = LevelEncoder::v1_streaming(max_level); + enc.put_n_with_observer(2, 1000, |_, _| {}); + + let mut ref_enc = LevelEncoder::v1_streaming(max_level); + ref_enc.put_with_observer(&[2; 1000], |_, _| {}); + assert_eq!(enc.consume(), ref_enc.consume()); + } +} diff --git a/parquet/src/file/metadata/memory.rs b/parquet/src/file/metadata/memory.rs index 30c10e7f2293..8a5937c43c65 100644 --- a/parquet/src/file/metadata/memory.rs +++ b/parquet/src/file/metadata/memory.rs @@ -18,7 +18,7 @@ //! Memory calculations for [`ParquetMetadata::memory_size`] //! //! [`ParquetMetadata::memory_size`]: crate::file::metadata::ParquetMetaData::memory_size -use crate::basic::{BoundaryOrder, ColumnOrder, Compression, Encoding, PageType}; +use crate::basic::{BoundaryOrder, ColumnOrder, CompressionCodec, Encoding, PageType}; use crate::data_type::private::ParquetValueType; use crate::file::metadata::{ ColumnChunkMetaData, FileMetaData, KeyValue, PageEncodingStats, ParquetPageEncodingStats, @@ -206,7 +206,7 @@ impl HeapSize for SortingColumn { 0 // no heap allocations } } -impl HeapSize for Compression { +impl HeapSize for CompressionCodec { fn heap_size(&self) -> usize { 0 // no heap allocations } diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index 156385ddaa70..646438d2e979 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -95,9 +95,13 @@ pub(crate) mod reader; pub(crate) mod thrift; mod writer; -use crate::basic::{EncodingMask, PageType}; +use crate::basic::{ + BoundaryOrder, ColumnOrder, Compression, CompressionCodec, Encoding, EncodingMask, PageType, + Type, +}; #[cfg(feature = "encryption")] use crate::encryption::decrypt::FileDecryptor; +use crate::errors::{ParquetError, Result}; #[cfg(feature = "encryption")] use crate::file::column_crypto_metadata::ColumnCryptoMetaData; pub(crate) use crate::file::metadata::memory::HeapSize; @@ -107,22 +111,15 @@ use crate::file::page_index::column_index::{ByteArrayColumnIndex, PrimitiveColum use crate::file::page_index::{column_index::ColumnIndexMetaData, offset_index::PageLocation}; use crate::file::statistics::Statistics; use crate::geospatial::statistics as geo_statistics; +use crate::parquet_thrift::{ + ElementType, FieldType, ReadThrift, ThriftCompactInputProtocol, ThriftCompactOutputProtocol, + WriteThrift, WriteThriftField, +}; use crate::schema::types::{ ColumnDescPtr, ColumnDescriptor, ColumnPath, SchemaDescPtr, SchemaDescriptor, Type as SchemaType, }; use crate::thrift_struct; -use crate::{ - basic::BoundaryOrder, - errors::{ParquetError, Result}, -}; -use crate::{ - basic::{ColumnOrder, Compression, Encoding, Type}, - parquet_thrift::{ - ElementType, FieldType, ReadThrift, ThriftCompactInputProtocol, - ThriftCompactOutputProtocol, WriteThrift, WriteThriftField, - }, -}; use crate::{ data_type::private::ParquetValueType, file::page_index::offset_index::OffsetIndexMetaData, }; @@ -814,7 +811,7 @@ pub struct ColumnChunkMetaData { file_path: Option, file_offset: i64, num_values: i64, - compression: Compression, + compression: CompressionCodec, total_compressed_size: i64, total_uncompressed_size: i64, data_page_offset: i64, @@ -1016,8 +1013,18 @@ impl ColumnChunkMetaData { self.num_values } - /// Compression for this column. + /// [`Compression`] for this column. + /// + /// This is a default value suitable for passing to [`WriterPropertiesBuilder::set_compression`]. + /// It is constructed from the `codec` field of the Parquet `ColumnMetaData` + /// + /// [`WriterPropertiesBuilder::set_compression`]: crate::file::properties::WriterPropertiesBuilder pub fn compression(&self) -> Compression { + self.compression.into() + } + + /// Returns the compression codec used when writing this column. + pub fn compression_codec(&self) -> CompressionCodec { self.compression } @@ -1234,7 +1241,7 @@ impl ColumnChunkMetaDataBuilder { file_path: None, file_offset: 0, num_values: 0, - compression: Compression::UNCOMPRESSED, + compression: CompressionCodec::UNCOMPRESSED, total_compressed_size: 0, total_uncompressed_size: 0, data_page_offset: 0, @@ -1285,8 +1292,14 @@ impl ColumnChunkMetaDataBuilder { self } - /// Sets compression. + /// Sets compression codec given a [`Compression`] configuration value. pub fn set_compression(mut self, value: Compression) -> Self { + self.0.compression = value.into(); + self + } + + /// Sets compression codec. + pub fn set_compression_codec(mut self, value: CompressionCodec) -> Self { self.0.compression = value; self } @@ -1820,7 +1833,7 @@ mod tests { )) .set_file_path("file_path".to_owned()) .set_num_values(1000) - .set_compression(Compression::SNAPPY) + .set_compression_codec(CompressionCodec::SNAPPY) .set_total_compressed_size(2000) .set_total_uncompressed_size(3000) .set_data_page_offset(4000) @@ -1860,7 +1873,7 @@ mod tests { )) .set_file_path("file_path".to_owned()) .set_num_values(1000) - .set_compression(Compression::SNAPPY) + .set_compression_codec(CompressionCodec::SNAPPY) .set_total_compressed_size(2000) .set_total_uncompressed_size(3000) .set_data_page_offset(4000) @@ -1903,7 +1916,7 @@ mod tests { [Encoding::PLAIN, Encoding::RLE].iter(), )) .set_num_values(1000) - .set_compression(Compression::SNAPPY) + .set_compression_codec(CompressionCodec::SNAPPY) .set_total_compressed_size(2000) .set_total_uncompressed_size(3000) .set_data_page_offset(4000) @@ -2034,9 +2047,9 @@ mod tests { .build(); #[cfg(not(feature = "encryption"))] - let base_expected_size = 2766; + let base_expected_size = 2734; #[cfg(feature = "encryption")] - let base_expected_size = 2934; + let base_expected_size = 2902; assert_eq!(parquet_meta.memory_size(), base_expected_size); @@ -2065,9 +2078,9 @@ mod tests { .build(); #[cfg(not(feature = "encryption"))] - let bigger_expected_size = 3192; + let bigger_expected_size = 3160; #[cfg(feature = "encryption")] - let bigger_expected_size = 3360; + let bigger_expected_size = 3328; // more set fields means more memory usage assert!(bigger_expected_size > base_expected_size); @@ -2114,7 +2127,7 @@ mod tests { .set_row_groups(row_group_meta.clone()) .build(); - let base_expected_size = 2058; + let base_expected_size = 2042; assert_eq!(parquet_meta_data.memory_size(), base_expected_size); let footer_key = "0123456789012345".as_bytes(); @@ -2140,7 +2153,7 @@ mod tests { .set_file_decryptor(Some(decryptor)) .build(); - let expected_size_with_decryptor = 3072; + let expected_size_with_decryptor = 3056; assert!(expected_size_with_decryptor > base_expected_size); assert_eq!( diff --git a/parquet/src/file/metadata/thrift/mod.rs b/parquet/src/file/metadata/thrift/mod.rs index ad8f6312c215..75fcad871e2f 100644 --- a/parquet/src/file/metadata/thrift/mod.rs +++ b/parquet/src/file/metadata/thrift/mod.rs @@ -35,8 +35,8 @@ use crate::file::{ }; use crate::{ basic::{ - ColumnOrder, Compression, ConvertedType, Encoding, EncodingMask, LogicalType, PageType, - Repetition, Type, + ColumnOrder, CompressionCodec, ConvertedType, Encoding, EncodingMask, LogicalType, + PageType, Repetition, Type, }, data_type::{ByteArray, FixedLenByteArray, Int96}, errors::{ParquetError, Result}, @@ -51,7 +51,7 @@ use crate::{ parquet_thrift::{ ElementType, FieldType, ReadThrift, ThriftCompactInputProtocol, ThriftCompactOutputProtocol, ThriftSliceInputProtocol, WriteThrift, WriteThriftField, - read_thrift_vec, + read_thrift_vec, validate_list_type, }, schema::types::{ ColumnDescriptor, SchemaDescriptor, TypePtr, num_nodes, parquet_schema_from_array, @@ -387,6 +387,9 @@ fn read_encoding_stats_as_mask<'a>( // read the vector of stats, setting mask bits for data pages let mut mask = 0i32; let list_ident = prot.read_list_begin()?; + // check for PageEncodingStats struct + validate_list_type(ElementType::Struct, &list_ident)?; + for _ in 0..list_ident.size { let pes = PageEncodingStats::read_thrift(prot)?; match pes.page_type { @@ -460,7 +463,7 @@ fn read_column_metadata<'a>( } // 3: path_in_schema is redundant 4 => { - column.compression = Compression::read_thrift(&mut *prot)?; + column.compression = CompressionCodec::read_thrift(&mut *prot)?; seen_mask |= COL_META_CODEC; } 5 => { @@ -652,6 +655,8 @@ fn read_row_group( match field_ident.id { 1 => { let list_ident = prot.read_list_begin()?; + // check for list of struct + validate_list_type(ElementType::Struct, &list_ident)?; if schema_descr.num_columns() != list_ident.size as usize { return Err(general_err!( "Column count mismatch. Schema has {} columns while Row Group has {}", @@ -801,6 +806,8 @@ pub(crate) fn parquet_metadata_from_bytes( } let schema_descr = schema_descr.as_ref().unwrap(); let list_ident = prot.read_list_begin()?; + // check for list of struct + validate_list_type(ElementType::Struct, &list_ident)?; let mut rg_vec = Vec::with_capacity(list_ident.size as usize); // Read row groups and handle ordinal assignment @@ -1333,10 +1340,15 @@ pub(super) fn serialize_column_meta_data( .encodings() .collect::>() .write_thrift_field(w, 2, 1)?; - let path = column_chunk.column_descr.path().parts(); - let path: Vec<&str> = path.iter().map(|v| v.as_str()).collect(); - path.write_thrift_field(w, 3, 2)?; - column_chunk.compression.write_thrift_field(w, 4, 3)?; + if w.write_path_in_schema() { + let path = column_chunk.column_descr.path().parts(); + let path: Vec<&str> = path.iter().map(|v| v.as_str()).collect(); + path.write_thrift_field(w, 3, 2)?; + column_chunk.compression.write_thrift_field(w, 4, 3)?; + } else { + column_chunk.compression.write_thrift_field(w, 4, 2)?; + } + column_chunk.num_values.write_thrift_field(w, 5, 4)?; column_chunk .total_uncompressed_size @@ -1406,6 +1418,8 @@ pub(super) fn serialize_column_meta_data( pub(super) struct FileMeta<'a> { pub(super) file_metadata: &'a crate::file::metadata::FileMetaData, pub(super) row_groups: &'a Vec, + // If true, then write the `path_in_schema` field in the ColumnMetaData struct. + pub(super) write_path_in_schema: bool, } // struct FileMetaData { @@ -1425,6 +1439,8 @@ impl<'a> WriteThrift for FileMeta<'a> { // needed for last_field_id w/o encryption #[allow(unused_assignments)] fn write_thrift(&self, writer: &mut ThriftCompactOutputProtocol) -> Result<()> { + writer.set_write_path_in_schema(self.write_path_in_schema); + self.file_metadata .version .write_thrift_field(writer, 1, 0)?; diff --git a/parquet/src/file/metadata/writer.rs b/parquet/src/file/metadata/writer.rs index 275b4ff28e56..cd5d617f9372 100644 --- a/parquet/src/file/metadata/writer.rs +++ b/parquet/src/file/metadata/writer.rs @@ -62,6 +62,7 @@ pub(crate) struct ThriftMetadataWriter<'a, W: Write> { created_by: Option, object_writer: MetadataObjectWriter, writer_version: i32, + write_path_in_schema: bool, } impl<'a, W: Write> ThriftMetadataWriter<'a, W> { @@ -259,6 +260,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { let file_meta = FileMeta { file_metadata: &file_metadata, row_groups: &row_groups, + write_path_in_schema: self.write_path_in_schema, }; // Write file metadata @@ -293,6 +295,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { row_groups: Vec, created_by: Option, writer_version: i32, + write_path_in_schema: bool, ) -> Self { Self { buf, @@ -304,6 +307,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { created_by, object_writer: Default::default(), writer_version, + write_path_in_schema, } } @@ -415,6 +419,7 @@ impl<'a, W: Write> ThriftMetadataWriter<'a, W> { pub struct ParquetMetaDataWriter<'a, W: Write> { buf: TrackedWrite, metadata: &'a ParquetMetaData, + write_path_in_schema: bool, } impl<'a, W: Write> ParquetMetaDataWriter<'a, W> { @@ -436,7 +441,20 @@ impl<'a, W: Write> ParquetMetaDataWriter<'a, W> { /// /// See example on the struct level documentation pub fn new_with_tracked(buf: TrackedWrite, metadata: &'a ParquetMetaData) -> Self { - Self { buf, metadata } + Self { + buf, + metadata, + write_path_in_schema: true, + } + } + + /// Set whether or not to write the `path_in_schema` field in the Thrift `ColumnMetaData` + /// struct. + pub fn with_write_path_in_schema(self, val: bool) -> Self { + Self { + write_path_in_schema: val, + ..self + } } /// Write the metadata to the buffer @@ -460,6 +478,7 @@ impl<'a, W: Write> ParquetMetaDataWriter<'a, W> { row_groups, created_by, file_metadata.version(), + self.write_path_in_schema, ); if let Some(column_indexes) = column_indexes { diff --git a/parquet/src/file/page_index/offset_index.rs b/parquet/src/file/page_index/offset_index.rs index b1e30dd4590c..06d21efb68c8 100644 --- a/parquet/src/file/page_index/offset_index.rs +++ b/parquet/src/file/page_index/offset_index.rs @@ -23,7 +23,7 @@ use std::io::Write; use crate::parquet_thrift::{ ElementType, FieldType, ReadThrift, ThriftCompactInputProtocol, ThriftCompactOutputProtocol, - WriteThrift, WriteThriftField, read_thrift_vec, + WriteThrift, WriteThriftField, read_thrift_vec, validate_list_type, }; use crate::{ errors::{ParquetError, Result}, @@ -90,6 +90,7 @@ impl OffsetIndexMetaData { // we have to do this manually because we want to use the fast PageLocation decoder let list_ident = prot.read_list_begin()?; + validate_list_type(ElementType::Struct, &list_ident)?; let mut page_locations = Vec::with_capacity(list_ident.size as usize); for _ in 0..list_ident.size { page_locations.push(read_page_location(prot)?); diff --git a/parquet/src/file/properties.rs b/parquet/src/file/properties.rs index 197c5d5c72a4..816973f83ca9 100644 --- a/parquet/src/file/properties.rs +++ b/parquet/src/file/properties.rs @@ -20,6 +20,7 @@ use crate::basic::{Compression, Encoding}; use crate::compression::{CodecOptions, CodecOptionsBuilder}; #[cfg(feature = "encryption")] use crate::encryption::encrypt::FileEncryptionProperties; +use crate::errors::{ParquetError, Result}; use crate::file::metadata::{KeyValue, SortingColumn}; use crate::schema::types::ColumnPath; use std::str::FromStr; @@ -69,6 +70,8 @@ pub const DEFAULT_OFFSET_INDEX_DISABLED: bool = false; pub const DEFAULT_COERCE_TYPES: bool = false; /// Default value for [`WriterProperties::data_page_v2_compression_ratio_threshold`] pub const DEFAULT_DATA_PAGE_V2_COMPRESSION_RATIO_THRESHOLD: f64 = 1.0; +/// Default value for [`WriterProperties::write_path_in_schema`] +pub const DEFAULT_WRITE_PATH_IN_SCHEMA: bool = true; /// Default minimum chunk size for content-defined chunking: 256 KiB. pub const DEFAULT_CDC_MIN_CHUNK_SIZE: usize = 256 * 1024; /// Default maximum chunk size for content-defined chunking: 1024 KiB. @@ -252,6 +255,7 @@ pub struct WriterProperties { statistics_truncate_length: Option, coerce_types: bool, content_defined_chunking: Option, + write_path_in_schema: bool, #[cfg(feature = "encryption")] pub(crate) file_encryption_properties: Option>, } @@ -437,6 +441,14 @@ impl WriterProperties { self.coerce_types } + /// Returns `true` if the `path_in_schema` field of the `ColumnMetaData` Thrift struct + /// should be written. + /// + /// For more details see [`WriterPropertiesBuilder::set_write_path_in_schema`] + pub fn write_path_in_schema(&self) -> bool { + self.write_path_in_schema + } + /// EXPERIMENTAL: Returns content-defined chunking options, or `None` if CDC is disabled. /// /// For more details see [`WriterPropertiesBuilder::set_content_defined_chunking`] @@ -592,6 +604,7 @@ pub struct WriterPropertiesBuilder { statistics_truncate_length: Option, coerce_types: bool, content_defined_chunking: Option, + write_path_in_schema: bool, #[cfg(feature = "encryption")] file_encryption_properties: Option>, } @@ -616,6 +629,7 @@ impl Default for WriterPropertiesBuilder { statistics_truncate_length: DEFAULT_STATISTICS_TRUNCATE_LENGTH, coerce_types: DEFAULT_COERCE_TYPES, content_defined_chunking: None, + write_path_in_schema: DEFAULT_WRITE_PATH_IN_SCHEMA, #[cfg(feature = "encryption")] file_encryption_properties: None, } @@ -670,6 +684,7 @@ impl WriterPropertiesBuilder { statistics_truncate_length: self.statistics_truncate_length, coerce_types: self.coerce_types, content_defined_chunking: self.content_defined_chunking, + write_path_in_schema: self.write_path_in_schema, #[cfg(feature = "encryption")] file_encryption_properties: self.file_encryption_properties, } @@ -885,6 +900,43 @@ impl WriterPropertiesBuilder { self } + /// EXPERIMENTAL: Should the writer emit the `path_in_schema` element of the + /// `ColumnMetaData` Thrift struct. Defaults to `true` via [`DEFAULT_WRITE_PATH_IN_SCHEMA`]. + /// + /// Because `path_in_schema` is a field on the `ColumnMetaData`, it is repeated + /// `num_columns * num_rowgroups` times. Compounding this is any level of nesting or + /// repetition in the schema. For instance, a top-level list column named `foo` will have + /// a `path_in_schema` of `["foo", "list", "element"]`. A list-of-struct is even worse, + /// because the necessary list wrapping is repeated for each element of the struct. A + /// file with a deeply nested schema and many row groups can have a large percentage of the + /// footer taken up by this field. For example, a file of 38 row groups with a schema containing + /// several lists of structs containing lists had 36% of the footer taken up by `path_in_schema`. + /// Removing this redundant information can greatly speed up footer parsing, which is particularly + /// important in scenarios where one does not wish to read the entire file (e.g. point + /// lookups). + /// + ///
+ /// + /// **WARNING:** + /// Setting this to `false` will break compatibility with Parquet readers that + /// still expect this field to be present. Virtually all Parquet readers (parquet-java, + /// Spark, arrow-cpp, pyarrow, pandas to name a few), with the exception + /// of the one in this crate, expect this field to be present, and will terminate execution + /// if it is not. This will continue to be the case unless/until the Parquet format + /// specification is explicitly changed to allow this field to be missing. As a consquence, + /// users should only set this to `false` if they have verified that any reader(s) they plan + /// to use can tolerate the absence of this field. + /// + /// For more context, see [GH-563]. + /// + ///
+ /// + /// [GH-563]: https://github.com/apache/parquet-format/issues/563 + pub fn set_write_path_in_schema(mut self, write_path_in_schema: bool) -> Self { + self.write_path_in_schema = write_path_in_schema; + self + } + /// EXPERIMENTAL: Sets content-defined chunking options, or disables CDC with `None`. /// /// When enabled, data page boundaries are determined by a rolling hash of the @@ -1068,10 +1120,10 @@ impl WriterPropertiesBuilder { /// * If the bloom filter is enabled previously then it is a no-op. /// /// * If the bloom filter is not enabled, default values for ndv and fpp - /// value are used used. See [`set_bloom_filter_ndv`] and + /// value are used used. See [`set_bloom_filter_max_ndv`] and /// [`set_bloom_filter_fpp`] to further adjust the ndv and fpp. /// - /// [`set_bloom_filter_ndv`]: Self::set_bloom_filter_ndv + /// [`set_bloom_filter_max_ndv`]: Self::set_bloom_filter_max_ndv /// [`set_bloom_filter_fpp`]: Self::set_bloom_filter_fpp pub fn set_bloom_filter_enabled(mut self, value: bool) -> Self { self.default_column_properties @@ -1103,11 +1155,17 @@ impl WriterPropertiesBuilder { /// been called. /// /// [`set_bloom_filter_enabled`]: Self::set_bloom_filter_enabled - pub fn set_bloom_filter_ndv(mut self, value: u64) -> Self { + pub fn set_bloom_filter_max_ndv(mut self, value: u64) -> Self { self.default_column_properties.set_bloom_filter_ndv(value); self } + /// Deprecated alias for [`Self::set_bloom_filter_max_ndv`]. + #[deprecated(since = "59.0.0", note = "Use `set_bloom_filter_max_ndv` instead")] + pub fn set_bloom_filter_ndv(self, value: u64) -> Self { + self.set_bloom_filter_max_ndv(value) + } + // ---------------------------------------------------------------------- // Setters for a specific column @@ -1206,10 +1264,11 @@ impl WriterPropertiesBuilder { self } - /// Sets the number of distinct values for bloom filter for a specific column. + /// Sets the maximum expected number of distinct values for bloom filter for + /// a specific column. /// - /// Takes precedence over [`Self::set_bloom_filter_ndv`]. - pub fn set_column_bloom_filter_ndv(mut self, col: ColumnPath, value: u64) -> Self { + /// Takes precedence over [`Self::set_bloom_filter_max_ndv`]. + pub fn set_column_bloom_filter_max_ndv(mut self, col: ColumnPath, value: u64) -> Self { self.get_mut_props(col).set_bloom_filter_ndv(value); self } @@ -1229,6 +1288,41 @@ impl WriterPropertiesBuilder { .set_data_page_v2_compression_ratio_threshold(value); self } + + /// Deprecated alias for [`Self::set_column_bloom_filter_max_ndv`]. + #[deprecated( + since = "59.0.0", + note = "Use `set_column_bloom_filter_max_ndv` instead" + )] + pub fn set_column_bloom_filter_ndv(self, col: ColumnPath, value: u64) -> Self { + self.set_column_bloom_filter_max_ndv(col, value) + } + + /// Sets the [`BloomFilterProperties`] for all columns, implicitly enabling + /// the bloom filter. + /// + /// Both `fpp` and `ndv` from `value` are treated as explicit and will not + /// be overridden by the build-time row-group-size NDV fallback. For + /// dynamic NDV sizing (resolved to `max_row_group_row_count` at build + /// time), use [`Self::set_bloom_filter_enabled`] or + /// [`Self::set_bloom_filter_fpp`] instead. + pub fn set_bloom_filter_properties(mut self, value: BloomFilterProperties) -> Self { + self.default_column_properties + .set_bloom_filter_properties(value); + self + } + + /// Sets the [`BloomFilterProperties`] for a specific column. + /// + /// Takes precedence over [`Self::set_bloom_filter_properties`]. + pub fn set_column_bloom_filter_properties( + mut self, + col: ColumnPath, + value: BloomFilterProperties, + ) -> Self { + self.get_mut_props(col).set_bloom_filter_properties(value); + self + } } impl From for WriterPropertiesBuilder { @@ -1253,6 +1347,7 @@ impl From for WriterPropertiesBuilder { statistics_truncate_length: props.statistics_truncate_length, coerce_types: props.coerce_types, content_defined_chunking: props.content_defined_chunking, + write_path_in_schema: props.write_path_in_schema, #[cfg(feature = "encryption")] file_encryption_properties: props.file_encryption_properties, } @@ -1316,6 +1411,29 @@ impl Default for EnabledStatistics { /// maintaining the target `fpp`. See [`Sbbf::fold_to_target_fpp`] for details on the /// folding algorithm. /// +/// # Example +/// +/// ```rust +/// # use parquet::{ +/// # file::properties::{BloomFilterProperties, WriterProperties}, +/// # schema::types::ColumnPath, +/// # }; +/// // Build a BloomFilterProperties via the builder, then apply it to one column. +/// let bf = BloomFilterProperties::builder() +/// .with_fpp(0.01) +/// .with_max_ndv(10_000) +/// .build(); +/// +/// let props = WriterProperties::builder() +/// .set_column_bloom_filter_properties(ColumnPath::from("user_id"), bf.clone()) +/// .build(); +/// +/// assert_eq!( +/// props.bloom_filter_properties(&ColumnPath::from("user_id")), +/// Some(&bf) +/// ); +/// ``` +/// /// [`Sbbf::fold_to_target_fpp`]: crate::bloom_filter::Sbbf::fold_to_target_fpp #[derive(Debug, Clone, PartialEq)] pub struct BloomFilterProperties { @@ -1332,7 +1450,7 @@ pub struct BloomFilterProperties { pub fpp: f64, /// Maximum expected number of distinct values. Defaults to [`DEFAULT_BLOOM_FILTER_NDV`]. /// - /// You should set this value by calling [`WriterPropertiesBuilder::set_bloom_filter_ndv`]. + /// You should set this value by calling [`WriterPropertiesBuilder::set_bloom_filter_max_ndv`]. /// /// When not explicitly set via the builder, this defaults to /// [`max_row_group_row_count`](WriterProperties::max_row_group_row_count) (resolved at @@ -1349,7 +1467,7 @@ pub struct BloomFilterProperties { /// of the number of distinct values in a row group, it is recommended to set this value explicitly /// rather than relying on the default dynamic sizing based on `max_row_group_row_count`. /// If you do set this value explicitly it is probably best to set it for each column - /// individually via [`WriterPropertiesBuilder::set_column_bloom_filter_ndv`] rather than globally, + /// individually via [`WriterPropertiesBuilder::set_column_bloom_filter_max_ndv`] rather than globally, /// since different columns may have different numbers of distinct values. pub ndv: u64, } @@ -1363,6 +1481,93 @@ impl Default for BloomFilterProperties { } } +impl BloomFilterProperties { + /// Returns a new [`BloomFilterPropertiesBuilder`] for constructing + /// [`BloomFilterProperties`] with custom values. + pub fn builder() -> BloomFilterPropertiesBuilder { + BloomFilterPropertiesBuilder::new() + } +} + +/// Builder for [`BloomFilterProperties`]. +/// +/// Use [`BloomFilterProperties::builder`] or [`BloomFilterPropertiesBuilder::new`] +/// as the entry point. +#[derive(Debug, Clone, Default)] +pub struct BloomFilterPropertiesBuilder { + fpp: Option, + ndv: Option, +} + +impl BloomFilterPropertiesBuilder { + /// Returns a new builder with no fields set. + /// + /// Equivalent to [`BloomFilterProperties::builder`]. + pub fn new() -> Self { + Self::default() + } + + /// Sets the target false positive probability. + /// + /// The value must be in `(0.0, 1.0)` exclusively; this is validated at + /// build time by [`Self::build`] / [`Self::try_build`]. When unset, the + /// default is `0.05` (5%, see [`DEFAULT_BLOOM_FILTER_FPP`]). + pub fn with_fpp(mut self, fpp: f64) -> Self { + self.fpp = Some(fpp); + self + } + + /// Sets the maximum expected number of distinct values used to size the + /// bloom filter before folding. + /// + /// When unset, the default is `1_048_576` (see [`DEFAULT_BLOOM_FILTER_NDV`]), + /// which at the default fpp of 5% reserves roughly 1 MiB per column for the + /// filter bitset, derived as follows: + /// + /// ```text + /// ndv = 1,048,576, fpp = 0.05 + /// 0.05^(1/8) ≈ 0.6877 + /// 1 - 0.6877 ≈ 0.3123 + /// ln(0.3123) ≈ -1.164 + /// num_bits = -8 * 1,048,576 / -1.164 ≈ 7,206,000 bits + /// ≈ 900,750 bytes (~900 KB) + /// next_power_of_two(900 KB) = 1 MiB (= 1,048,576 bytes) + /// ``` + pub fn with_max_ndv(mut self, ndv: u64) -> Self { + self.ndv = Some(ndv); + self + } + + /// Builds [`BloomFilterProperties`]. + /// + /// Panics if the configured `fpp` is not in `(0.0, 1.0)` exclusive. + /// Use [`Self::try_build`] for a non-panicking alternative. + pub fn build(self) -> BloomFilterProperties { + self.try_build().unwrap_or_else(|e| panic!("{e}")) + } + + /// Builds [`BloomFilterProperties`], returning an error instead of + /// panicking when the configured `fpp` is not in `(0.0, 1.0)` exclusive. + pub fn try_build(self) -> Result { + let fpp = self.fpp.unwrap_or(DEFAULT_BLOOM_FILTER_FPP); + validate_bloom_filter_fpp(fpp).map_err(ParquetError::General)?; + let ndv = self.ndv.unwrap_or(DEFAULT_BLOOM_FILTER_NDV); + Ok(BloomFilterProperties { fpp, ndv }) + } +} + +/// Single source of truth for the bloom filter fpp range check, shared by +/// [`ColumnProperties::set_bloom_filter_fpp`] (panic path) and +/// [`BloomFilterPropertiesBuilder::try_build`] (Result path). +fn validate_bloom_filter_fpp(fpp: f64) -> std::result::Result<(), String> { + if !(fpp > 0.0 && fpp < 1.0) { + return Err(format!( + "fpp must be between 0.0 and 1.0 exclusive, got {fpp}" + )); + } + Ok(()) +} + /// Container for column properties that can be changed as part of writer. /// /// If a field is `None`, it means that no specific value has been set for this column, @@ -1448,11 +1653,9 @@ impl ColumnProperties { /// /// Panics if the `value` is not between 0 and 1 exclusive fn set_bloom_filter_fpp(&mut self, value: f64) { - assert!( - value > 0. && value < 1.0, - "fpp must be between 0 and 1 exclusive, got {value}" - ); - + if let Err(msg) = validate_bloom_filter_fpp(value) { + panic!("{msg}"); + } self.bloom_filter_properties .get_or_insert_with(Default::default) .fpp = value; @@ -1467,6 +1670,17 @@ impl ColumnProperties { self.bloom_filter_ndv_is_set = true; } + /// Sets the bloom filter properties for this column from a fully-built + /// [`BloomFilterProperties`], implicitly enabling the bloom filter. + /// + /// Both `fpp` and `ndv` from `value` are treated as explicit, so the + /// build-time row-group-size NDV fallback in + /// [`WriterPropertiesBuilder::build`] will not override them. + fn set_bloom_filter_properties(&mut self, value: BloomFilterProperties) { + self.bloom_filter_properties = Some(value); + self.bloom_filter_ndv_is_set = true; + } + /// Sets the Data Page v2 compression ratio threshold for this column. /// /// # Panics @@ -1787,7 +2001,7 @@ mod tests { .set_column_dictionary_enabled(ColumnPath::from("col"), true) .set_column_statistics_enabled(ColumnPath::from("col"), EnabledStatistics::Chunk) .set_column_bloom_filter_enabled(ColumnPath::from("col"), true) - .set_column_bloom_filter_ndv(ColumnPath::from("col"), 100_u64) + .set_column_bloom_filter_max_ndv(ColumnPath::from("col"), 100_u64) .set_column_bloom_filter_fpp(ColumnPath::from("col"), 0.1) .build(); @@ -1909,7 +2123,7 @@ mod tests { ); assert_eq!( WriterProperties::builder() - .set_bloom_filter_ndv(100) + .set_bloom_filter_max_ndv(100) .build() .bloom_filter_properties(&ColumnPath::from("col")), Some(&BloomFilterProperties { @@ -1957,6 +2171,30 @@ mod tests { .build(); } + #[test] + #[allow(deprecated)] + fn test_writer_properties_deprecated_bloom_filter_ndv_setters_still_work() { + let col = ColumnPath::from("col"); + let props = WriterProperties::builder() + .set_bloom_filter_ndv(100) + .set_column_bloom_filter_ndv(col.clone(), 200) + .build(); + assert_eq!( + props.bloom_filter_properties(&ColumnPath::from("other")), + Some(&BloomFilterProperties { + fpp: DEFAULT_BLOOM_FILTER_FPP, + ndv: 100, + }) + ); + assert_eq!( + props.bloom_filter_properties(&col), + Some(&BloomFilterProperties { + fpp: DEFAULT_BLOOM_FILTER_FPP, + ndv: 200, + }) + ); + } + #[test] fn test_writer_properties_column_dictionary_page_size_limit() { let props = WriterProperties::builder() @@ -2073,4 +2311,141 @@ mod tests { assert_eq!(custom, custom); assert_ne!(opts, custom); } + + #[test] + fn test_bloom_filter_builder_default() { + let props = BloomFilterProperties::builder().build(); + assert_eq!(props.fpp, DEFAULT_BLOOM_FILTER_FPP); + assert_eq!(props.ndv, DEFAULT_BLOOM_FILTER_NDV); + assert_eq!(props, BloomFilterProperties::default()); + assert_eq!( + BloomFilterPropertiesBuilder::new().build(), + BloomFilterProperties::default() + ); + } + + #[test] + fn test_bloom_filter_builder_explicit_fpp() { + let props = BloomFilterProperties::builder().with_fpp(0.01).build(); + assert_eq!(props.fpp, 0.01); + assert_eq!(props.ndv, DEFAULT_BLOOM_FILTER_NDV); + } + + #[test] + fn test_bloom_filter_builder_explicit_ndv() { + let props = BloomFilterProperties::builder().with_max_ndv(1000).build(); + assert_eq!(props.fpp, DEFAULT_BLOOM_FILTER_FPP); + assert_eq!(props.ndv, 1000); + } + + #[test] + fn test_bloom_filter_builder_validates_fpp() { + for wrong_val in [0.0_f64, 1.0, -0.5, 2.0] { + let result = std::panic::catch_unwind(|| { + BloomFilterProperties::builder().with_fpp(wrong_val).build() + }); + assert!( + result.is_err(), + "with_fpp({wrong_val}).build() should reject value outside (0, 1)" + ); + } + } + + #[test] + fn test_bloom_filter_builder_try_build_validates_fpp() { + for wrong_val in [0.0_f64, 1.0, -0.5, 2.0] { + let result = BloomFilterProperties::builder() + .with_fpp(wrong_val) + .try_build(); + assert!( + result.is_err(), + "try_build() should return Err for fpp outside (0, 1)" + ); + } + + let ok = BloomFilterProperties::builder() + .with_fpp(0.01) + .with_max_ndv(1000) + .try_build() + .expect("valid fpp should yield Ok"); + assert_eq!(ok.fpp, 0.01); + assert_eq!(ok.ndv, 1000); + } + + #[test] + fn test_column_specific_implicit_ndv_uses_row_group_size() { + let custom_row_group_size: usize = 7777; + let col = ColumnPath::from("col"); + let props = WriterProperties::builder() + .set_max_row_group_row_count(Some(custom_row_group_size)) + .set_column_bloom_filter_enabled(col.clone(), true) + .build(); + let bf = props + .bloom_filter_properties(&col) + .expect("bloom filter should be enabled for col"); + + assert_eq!(bf.ndv, custom_row_group_size as u64); + assert_eq!(bf.fpp, DEFAULT_BLOOM_FILTER_FPP); + } + + #[test] + fn test_set_bloom_filter_properties_applied_globally() { + let bf = BloomFilterProperties::builder() + .with_fpp(0.01) + .with_max_ndv(500) + .build(); + let props = WriterProperties::builder() + .set_bloom_filter_properties(bf.clone()) + .build(); + + assert_eq!( + props.bloom_filter_properties(&ColumnPath::from("a")), + Some(&bf), + ); + assert_eq!( + props.bloom_filter_properties(&ColumnPath::from("b")), + Some(&bf), + ); + } + + #[test] + fn test_set_column_bloom_filter_properties_overrides_global() { + let global = BloomFilterProperties::builder() + .with_fpp(0.01) + .with_max_ndv(500) + .build(); + let tailored = BloomFilterProperties::builder() + .with_fpp(0.02) + .with_max_ndv(1000) + .build(); + + let col = ColumnPath::from("col"); + let props = WriterProperties::builder() + .set_bloom_filter_properties(global.clone()) + .set_column_bloom_filter_properties(col.clone(), tailored.clone()) + .build(); + + assert_eq!(props.bloom_filter_properties(&col), Some(&tailored)); + assert_eq!( + props.bloom_filter_properties(&ColumnPath::from("other")), + Some(&global) + ); + } + + #[test] + fn test_set_bloom_filter_properties_preserve_explicit_ndv() { + let bf = BloomFilterProperties::builder().with_max_ndv(42).build(); + let props = WriterProperties::builder() + .set_max_row_group_row_count(Some(99_999)) + .set_bloom_filter_properties(bf) + .build(); + let result = props + .bloom_filter_properties(&ColumnPath::from("col")) + .expect("bloom filter should be enabled"); + + assert_eq!( + result.ndv, 42, + "explicit ndv must not be overridden by row-group-size fallback" + ); + } } diff --git a/parquet/src/file/serialized_reader.rs b/parquet/src/file/serialized_reader.rs index 254ccb779a4a..4b71e3c14e6d 100644 --- a/parquet/src/file/serialized_reader.rs +++ b/parquet/src/file/serialized_reader.rs @@ -2117,7 +2117,7 @@ mod tests { let ret = SerializedFileReader::new(Bytes::copy_from_slice(&data)); assert_eq!( ret.err().unwrap().to_string(), - "Parquet error: Received empty union from remote ColumnOrder" + "Parquet error: Expected list element type of Struct but got List" ); } diff --git a/parquet/src/file/writer.rs b/parquet/src/file/writer.rs index 7d69904451d3..ec4d02a47d79 100644 --- a/parquet/src/file/writer.rs +++ b/parquet/src/file/writer.rs @@ -345,12 +345,14 @@ impl SerializedFileWriter { let column_indexes = std::mem::take(&mut self.column_indexes); let offset_indexes = std::mem::take(&mut self.offset_indexes); + let write_path_in_schema = self.props.write_path_in_schema(); let mut encoder = ThriftMetadataWriter::new( &mut self.buf, &self.descr, row_groups, Some(self.props.created_by().to_string()), self.props.writer_version().as_num(), + write_path_in_schema, ); #[cfg(feature = "encryption")] @@ -716,7 +718,7 @@ impl<'a, W: Write + Send> SerializedRowGroupWriter<'a, W> { let map_offset = |x| x - src_offset + write_offset as i64; let mut builder = ColumnChunkMetaData::builder(metadata.column_descr_ptr()) - .set_compression(metadata.compression()) + .set_compression_codec(metadata.compression_codec()) .set_encodings_mask(*metadata.encodings_mask()) .set_total_compressed_size(metadata.compressed_size()) .set_total_uncompressed_size(metadata.uncompressed_size()) @@ -1595,7 +1597,7 @@ mod tests { let desc = ColumnDescriptor::new(Arc::new(t), 0, 0, ColumnPath::new(vec![])); let meta = ColumnChunkMetaData::builder(Arc::new(desc)) - .set_compression(codec) + .set_compression_codec(codec.into()) .set_total_compressed_size(reader.len() as i64) .set_num_values(total_num_values) .build() diff --git a/parquet/src/parquet_thrift.rs b/parquet/src/parquet_thrift.rs index 6c82a0bf2c07..b9d69b9d52e8 100644 --- a/parquet/src/parquet_thrift.rs +++ b/parquet/src/parquet_thrift.rs @@ -36,6 +36,7 @@ use crate::{ write_thrift_field, }; use std::io::Error; +use std::num::TryFromIntError; use std::str::Utf8Error; #[derive(Debug)] @@ -46,6 +47,7 @@ pub(crate) enum ThriftProtocolError { InvalidElementType(u8), FieldDeltaOverflow { field_delta: u8, last_field_id: i16 }, InvalidBoolean(u8), + IntegerOverflow, Utf8Error, SkipDepth(FieldType), SkipUnsupportedType(FieldType), @@ -70,6 +72,9 @@ impl From for ParquetError { ThriftProtocolError::InvalidBoolean(value) => { general_err!("cannot convert {} into bool", value) } + ThriftProtocolError::IntegerOverflow => { + general_err!("integer overflow decoding thrift value") + } ThriftProtocolError::Utf8Error => general_err!("invalid utf8"), ThriftProtocolError::SkipDepth(field_type) => { general_err!("cannot parse past {:?}", field_type) @@ -94,6 +99,13 @@ impl From for ThriftProtocolError { } } +impl From for ThriftProtocolError { + fn from(_: TryFromIntError) -> Self { + // ignore error payload to reduce the size of ThriftProtocolError + Self::IntegerOverflow + } +} + pub type ThriftProtocolResult = Result; /// Wrapper for thrift `double` fields. This is used to provide @@ -317,7 +329,13 @@ pub(crate) trait ThriftCompactInputProtocol<'a> { // high bits set high if count and type encoded separately possible_element_count as i32 } else { - self.read_vlq()? as _ + // The list size on the wire is an unsigned varint, but we represent + // it as `i32` (matching Java's `int` and the Thrift schema). + // A varint that decodes above `i32::MAX` is malformed input — reject + // it here at the protocol layer rather than letting the cast wrap + // into a negative size that downstream allocation code has to + // re-validate. + i32::try_from(self.read_vlq()?)? }; Ok(ListIdentifier { @@ -684,9 +702,10 @@ impl<'a, R: ThriftCompactInputProtocol<'a>> ReadThrift<'a, R> for &'a [u8] { pub(crate) fn read_thrift_vec<'a, T, R>(prot: &mut R) -> Result> where R: ThriftCompactInputProtocol<'a>, - T: ReadThrift<'a, R>, + T: ReadThrift<'a, R> + WriteThrift, { let list_ident = prot.read_list_begin()?; + validate_list_type(T::ELEMENT_TYPE, &list_ident)?; let mut res = Vec::with_capacity(list_ident.size as usize); for _ in 0..list_ident.size { let val = T::read_thrift(prot)?; @@ -695,6 +714,17 @@ where Ok(res) } +pub(crate) fn validate_list_type(expected: ElementType, got: &ListIdentifier) -> Result<()> { + if got.element_type != expected { + return Err(general_err!( + "Expected list element type of {:?} but got {:?}", + expected, + got.element_type + )); + } + Ok(()) +} + ///////////////////////// // thrift compact output @@ -708,12 +738,29 @@ where /// [compact output]: https://github.com/apache/thrift/blob/master/doc/specs/thrift-compact-protocol.md pub(crate) struct ThriftCompactOutputProtocol { writer: W, + write_path_in_schema: bool, } impl ThriftCompactOutputProtocol { /// Create a new `ThriftCompactOutputProtocol` wrapping the byte sink `writer`. pub(crate) fn new(writer: W) -> Self { - Self { writer } + Self { + writer, + write_path_in_schema: true, + } + } + + // TODO(ets): at some point there should probably be a properties object + // to control aspects of thrift output. But since this is the only option to date + // I'm choosing a simpler API. + /// Control the writing of the `path_in_schema` element of the `ColumnMetaData` + pub(crate) fn set_write_path_in_schema(&mut self, val: bool) { + self.write_path_in_schema = val; + } + + /// Indicate whether or not to emit `path_in_schema`. + pub(crate) fn write_path_in_schema(&self) -> bool { + self.write_path_in_schema } /// Write a single byte to the output stream. @@ -1106,4 +1153,34 @@ pub(crate) mod tests { assert_eq!(header.size, 0); assert_eq!(header.element_type, ElementType::Byte); } + + /// A Thrift list header whose `size` varint decodes above `i32::MAX` + /// must be rejected at the protocol layer rather than wrapping into a + /// negative `i32` and being smuggled into downstream allocation code. + #[test] + fn test_read_list_begin_size_above_i32_max_returns_err() { + // List header: element_type=8 (Binary), 0xF=follow-up varint. + // Varint 80 80 80 80 08 decodes to 0x8000_0000 = i32::MAX + 1. + let mut data: Vec = vec![0xF8]; + data.extend_from_slice(&[0x80, 0x80, 0x80, 0x80, 0x08]); + let mut prot = ThriftSliceInputProtocol::new(&data); + let result = prot.read_list_begin(); + assert!(result.is_err(), "expected error, got {result:?}"); + } + + #[test] + fn test_read_list_wrong_type() { + // list header: 4 elements of `Boolean` + let data = [0x42, 0x01]; + let mut prot = ThriftSliceInputProtocol::new(&data); + // try to read as list + let result = read_thrift_vec::(&mut prot); + println!("{result:?}"); + assert!( + result + .unwrap_err() + .to_string() + .contains("Expected list element type of I32 but got Bool") + ); + } } diff --git a/parquet/src/schema/printer.rs b/parquet/src/schema/printer.rs index 4ad3b6b93e00..68398005b6a5 100644 --- a/parquet/src/schema/printer.rs +++ b/parquet/src/schema/printer.rs @@ -177,11 +177,7 @@ fn print_column_chunk_metadata(out: &mut dyn io::Write, cc_metadata: &ColumnChun writeln!(out, "file path: {file_path_str}"); writeln!(out, "file offset: {}", cc_metadata.file_offset()); writeln!(out, "num of values: {}", cc_metadata.num_values()); - writeln!( - out, - "compression: {}", - cc_metadata.compression().codec_to_string() - ); + writeln!(out, "compression: {}", cc_metadata.compression_codec()); writeln!( out, "total compressed size (in bytes): {}", diff --git a/parquet/src/schema/types.rs b/parquet/src/schema/types.rs index 2c63da74df47..0d504e16fc28 100644 --- a/parquet/src/schema/types.rs +++ b/parquet/src/schema/types.rs @@ -1401,7 +1401,7 @@ fn schema_from_array_helper<'a>( Some(n) => { let repetition = element.repetition_type; - let mut fields = Vec::with_capacity(n as usize); + let mut fields = Vec::with_capacity(usize::try_from(n)?); let mut next_index = index + 1; for _ in 0..n { let child_result = schema_from_array_helper(elements, num_elements, next_index)?; @@ -2517,4 +2517,22 @@ mod tests { let result_schema = parquet_schema_from_array(thrift_schema).unwrap(); assert_eq!(result_schema, expected_schema); } + + #[test] + fn test_parquet_schema_from_array_rejects_negative_num_children() { + let elements = vec![SchemaElement { + r#type: None, + type_length: None, + repetition_type: Some(Repetition::REQUIRED), + name: "schema", + num_children: Some(-1), + converted_type: None, + scale: None, + precision: None, + field_id: None, + logical_type: None, + }]; + let result = parquet_schema_from_array(elements); + assert!(result.unwrap_err().to_string().contains("Integer overflow")); + } } diff --git a/parquet/tests/arrow_reader/bad_data.rs b/parquet/tests/arrow_reader/bad_data.rs index d9b0d89e2c5f..56fddf505dac 100644 --- a/parquet/tests/arrow_reader/bad_data.rs +++ b/parquet/tests/arrow_reader/bad_data.rs @@ -98,7 +98,7 @@ fn test_arrow_gh_41317() { let err = read_file("ARROW-GH-41317.parquet").unwrap_err(); assert_eq!( err.to_string(), - "External: Parquet argument error: Parquet error: StructArrayReader out of sync in read_records, expected 5 read, got 2" + "Parquet error: Expected list element type of I32 but got I16" ); }