From 4282a1ac93763abcd8086e97361691ea7c8efbb5 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 22 Jul 2026 09:16:36 -0400 Subject: [PATCH 1/3] add 256bit data file key test, bump arrow-rs to 58.4 for this feature --- Cargo.lock | 74 ++++++++++----------- Cargo.toml | 20 +++--- crates/iceberg/src/arrow/reader/pipeline.rs | 20 +++++- 3 files changed, 64 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ead0b2e13c..87b228a871 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -227,9 +227,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "arrow" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" +checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" dependencies = [ "arrow-arith", "arrow-array", @@ -249,9 +249,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" +checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" dependencies = [ "arrow-array", "arrow-buffer", @@ -263,9 +263,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" dependencies = [ "ahash", "arrow-buffer", @@ -306,9 +306,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" dependencies = [ "bytes", "half", @@ -318,9 +318,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" dependencies = [ "arrow-array", "arrow-buffer", @@ -340,9 +340,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" +checksum = "af0dd6d90d1955e9f9a014c1e563ee8aeffc21909085d25623e1da44d96eca26" dependencies = [ "arrow-array", "arrow-cast", @@ -355,9 +355,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" dependencies = [ "arrow-buffer", "arrow-schema", @@ -368,9 +368,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" dependencies = [ "arrow-array", "arrow-buffer", @@ -384,9 +384,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" +checksum = "b8a96aed3931c076adee39ec2a40d8219fc7f09e79bcdaca1df16272993e1e14" dependencies = [ "arrow-array", "arrow-buffer", @@ -409,9 +409,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" dependencies = [ "arrow-array", "arrow-buffer", @@ -422,9 +422,9 @@ dependencies = [ [[package]] name = "arrow-pyarrow" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29abdf672a81c1aeb57fd2661457f9918964d49aed0e9f18932535f2a9e49ce" +checksum = "3ffb9be5a873590f825aef50df20e0f8dff5fd42a77058e28bbe7bd44bb53dec" dependencies = [ "arrow-array", "arrow-data", @@ -434,9 +434,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" dependencies = [ "arrow-array", "arrow-buffer", @@ -447,9 +447,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" dependencies = [ "bitflags 2.13.0", "serde_core", @@ -458,9 +458,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" dependencies = [ "ahash", "arrow-array", @@ -472,9 +472,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" +checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" dependencies = [ "arrow-array", "arrow-buffer", @@ -1483,7 +1483,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -3718,7 +3718,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.4", "system-configuration", "tokio", "tower-service", @@ -5435,9 +5435,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.3.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" +checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704" dependencies = [ "ahash", "arrow-array", @@ -5923,7 +5923,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.4", "thiserror 2.0.18", "tokio", "tracing", @@ -5962,7 +5962,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.4", "tracing", "windows-sys 0.61.2", ] @@ -8541,7 +8541,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c7315a6d13..a394ec107c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,15 +49,15 @@ aes-gcm = "0.10" anyhow = "1.0.72" apache-avro = { version = "0.21", features = ["snappy", "zstandard"] } array-init = "2" -arrow = "58" -arrow-arith = "58" -arrow-array = "58" -arrow-buffer = "58" -arrow-cast = "58" -arrow-ord = "58" -arrow-schema = "58" -arrow-select = "58" -arrow-string = "58" +arrow = "58.4" +arrow-arith = "58.4" +arrow-array = "58.4" +arrow-buffer = "58.4" +arrow-cast = "58.4" +arrow-ord = "58.4" +arrow-schema = "58.4" +arrow-select = "58.4" +arrow-string = "58.4" as-any = "0.3.2" async-trait = "0.1.89" aws-config = "1.8.7" @@ -121,7 +121,7 @@ murmur3 = "0.5.2" once_cell = "1.20" opendal = "0.57" ordered-float = "4" -parquet = "58" +parquet = "58.4" pilota = "0.11.10" pretty_assertions = "1.4" pyo3 = "0.28" diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index ccc7679ed0..cfd7eca855 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -737,9 +737,10 @@ mod tests { } } - #[tokio::test] - async fn test_read_encrypted_parquet() { - let encryption_key = b"0123456789abcdef"; + /// Writes a single-column Parquet file encrypted with `encryption_key`, then reads it + /// back through `ArrowReader` and asserts the round-tripped values. The key length + /// selects the AES-GCM variant in arrow-rs (16 -> AES-128, 32 -> AES-256). + async fn assert_encrypted_parquet_roundtrip(encryption_key: &[u8]) { let aad_prefix = b"aad_prefix"; let schema = Arc::new( @@ -807,6 +808,19 @@ mod tests { assert_eq!(ids.values(), &[10, 20, 30]); } + #[tokio::test] + async fn test_read_encrypted_parquet_aes_128() { + // 16-byte key -> AES-128-GCM. + assert_encrypted_parquet_roundtrip(b"0123456789abcdef").await; + } + + #[tokio::test] + async fn test_read_encrypted_parquet_aes_256() { + // 32-byte key -> AES-256-GCM. Requires arrow-rs >= 58.4.0; + // 58.3.0 and earlier only supported AES-128 for Parquet modular encryption. + assert_encrypted_parquet_roundtrip(b"0123456789abcdef0123456789abcdef").await; + } + #[tokio::test] async fn test_read_encrypted_parquet_without_key_metadata_fails() { let encryption_key = b"0123456789abcdef"; From b338c4590c689a9e44ac3096b9be76b4de8f6973 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 22 Jul 2026 09:20:21 -0400 Subject: [PATCH 2/3] clean up comments --- crates/iceberg/src/arrow/reader/pipeline.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index cfd7eca855..0ca8d8be78 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -810,14 +810,12 @@ mod tests { #[tokio::test] async fn test_read_encrypted_parquet_aes_128() { - // 16-byte key -> AES-128-GCM. assert_encrypted_parquet_roundtrip(b"0123456789abcdef").await; } #[tokio::test] async fn test_read_encrypted_parquet_aes_256() { - // 32-byte key -> AES-256-GCM. Requires arrow-rs >= 58.4.0; - // 58.3.0 and earlier only supported AES-128 for Parquet modular encryption. + // arrow-rs only added AES-256 for Parquet modular encryption in 58.4.0. assert_encrypted_parquet_roundtrip(b"0123456789abcdef0123456789abcdef").await; } From fcfa7b8f9ec0f5bb6f5f3190188ef25a0a30e576 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 22 Jul 2026 09:23:56 -0400 Subject: [PATCH 3/3] clean up comments --- crates/iceberg/src/arrow/reader/pipeline.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs index 0ca8d8be78..579f047266 100644 --- a/crates/iceberg/src/arrow/reader/pipeline.rs +++ b/crates/iceberg/src/arrow/reader/pipeline.rs @@ -815,7 +815,6 @@ mod tests { #[tokio::test] async fn test_read_encrypted_parquet_aes_256() { - // arrow-rs only added AES-256 for Parquet modular encryption in 58.4.0. assert_encrypted_parquet_roundtrip(b"0123456789abcdef0123456789abcdef").await; }