From c1ece6746df39ad0e5676b9393035e46bc88549a Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Thu, 30 Apr 2026 10:24:59 -0700 Subject: [PATCH 1/6] add CompressionCodec thrift enum --- parquet/src/basic.rs | 97 ++++++++++++------------- parquet/src/bin/parquet-layout.rs | 20 ++--- parquet/src/column/writer/mod.rs | 2 +- parquet/src/file/metadata/memory.rs | 4 +- parquet/src/file/metadata/mod.rs | 46 ++++++------ parquet/src/file/metadata/thrift/mod.rs | 6 +- parquet/src/file/serialized_reader.rs | 2 +- parquet/src/file/writer.rs | 2 +- parquet/src/schema/printer.rs | 2 +- 9 files changed, 86 insertions(+), 95 deletions(-) diff --git a/parquet/src/basic.rs b/parquet/src/basic.rs index ba8ffc2e92c3..133fd06cb54a 100644 --- a/parquet/src/basic.rs +++ b/parquet/src/basic.rs @@ -798,6 +798,28 @@ fn i32_to_encoding(val: i32) -> Encoding { // ---------------------------------------------------------------------- // Mirrors thrift enum `CompressionCodec` +thrift_enum!( +/// Supported compression algorithms. +/// +/// Codecs added in format version X.Y can be read by readers based on X.Y and later. +/// Codec support may vary between readers based on the format version and +/// libraries available at runtime. +/// +/// See [Compression.md] for a detailed specification of these algorithms. +/// +/// [Compression.md]: https://github.com/apache/parquet-format/blob/master/Compression.md +enum CompressionCodec { + UNCOMPRESSED = 0; + SNAPPY = 1; + GZIP = 2; + LZO = 3; + BROTLI = 4; // Added in 2.4 + LZ4 = 5; // DEPRECATED (Added in 2.4) + ZSTD = 6; // Added in 2.4 + LZ4_RAW = 7; // Added in 2.9 +} +); + /// Supported block compression algorithms. /// /// Block compression can yield non-trivial improvements to storage efficiency at the expense @@ -834,51 +856,33 @@ pub enum Compression { LZ4_RAW, } -impl<'a, R: ThriftCompactInputProtocol<'a>> ReadThrift<'a, R> for Compression { - fn read_thrift(prot: &mut R) -> Result { - let val = prot.read_i32()?; - Ok(match val { - 0 => Self::UNCOMPRESSED, - 1 => Self::SNAPPY, - 2 => Self::GZIP(Default::default()), - 3 => Self::LZO, - 4 => Self::BROTLI(Default::default()), - 5 => Self::LZ4, - 6 => Self::ZSTD(Default::default()), - 7 => Self::LZ4_RAW, - _ => return Err(general_err!("Unexpected CompressionCodec {}", val)), - }) - } -} - -// TODO(ets): explore replacing this with a thrift_enum!(ThriftCompression) for the serialization -// and then provide `From` impls to convert back and forth. This is necessary due to the addition -// of compression level to some variants. -impl WriteThrift for Compression { - const ELEMENT_TYPE: ElementType = ElementType::I32; - - fn write_thrift(&self, writer: &mut ThriftCompactOutputProtocol) -> Result<()> { - let id: i32 = match *self { - Self::UNCOMPRESSED => 0, - Self::SNAPPY => 1, - Self::GZIP(_) => 2, - Self::LZO => 3, - Self::BROTLI(_) => 4, - Self::LZ4 => 5, - Self::ZSTD(_) => 6, - Self::LZ4_RAW => 7, - }; - writer.write_i32(id) +impl From for Compression { + fn from(value: CompressionCodec) -> Self { + match value { + CompressionCodec::UNCOMPRESSED => Compression::UNCOMPRESSED, + CompressionCodec::SNAPPY => Compression::SNAPPY, + CompressionCodec::GZIP => Compression::GZIP(Default::default()), + CompressionCodec::LZO => Compression::LZO, + CompressionCodec::BROTLI => Compression::BROTLI(Default::default()), + CompressionCodec::LZ4 => Compression::LZ4, + CompressionCodec::ZSTD => Compression::ZSTD(Default::default()), + CompressionCodec::LZ4_RAW => Compression::LZ4_RAW, + } } } -write_thrift_field!(Compression, FieldType::I32); - -impl Compression { - /// Returns the codec type of this compression setting as a string, without the compression - /// level. - pub(crate) fn codec_to_string(self) -> String { - format!("{self:?}").split('(').next().unwrap().to_owned() +impl From for CompressionCodec { + fn from(value: Compression) -> Self { + match value { + Compression::UNCOMPRESSED => CompressionCodec::UNCOMPRESSED, + Compression::SNAPPY => CompressionCodec::SNAPPY, + Compression::GZIP(_) => CompressionCodec::GZIP, + Compression::LZO => CompressionCodec::LZO, + Compression::BROTLI(_) => CompressionCodec::BROTLI, + Compression::LZ4 => CompressionCodec::LZ4, + Compression::ZSTD(_) => CompressionCodec::ZSTD, + Compression::LZ4_RAW => CompressionCodec::LZ4_RAW, + } } } @@ -2139,15 +2143,6 @@ mod tests { assert_eq!(Encoding::RLE_DICTIONARY.to_string(), "RLE_DICTIONARY"); } - #[test] - fn test_compression_codec_to_string() { - assert_eq!(Compression::UNCOMPRESSED.codec_to_string(), "UNCOMPRESSED"); - assert_eq!( - Compression::ZSTD(ZstdLevel::default()).codec_to_string(), - "ZSTD" - ); - } - #[test] fn test_display_compression() { assert_eq!(Compression::UNCOMPRESSED.to_string(), "UNCOMPRESSED"); diff --git a/parquet/src/bin/parquet-layout.rs b/parquet/src/bin/parquet-layout.rs index 007f93517d96..3a95c6178a0e 100644 --- a/parquet/src/bin/parquet-layout.rs +++ b/parquet/src/bin/parquet-layout.rs @@ -48,7 +48,7 @@ use parquet::file::metadata::ParquetMetaDataReader; use serde::Serialize; use thrift::protocol::TCompactInputProtocol; -use parquet::basic::Compression; +use parquet::basic::CompressionCodec; use parquet::errors::Result; use parquet::file::reader::ChunkReader; #[allow(deprecated)] @@ -225,16 +225,16 @@ fn read_page_header(reader: &C, offset: u64) -> Result<(usize, P } /// Returns a string representation for a given compression -fn compression(compression: Compression) -> Option<&'static str> { +fn compression(compression: CompressionCodec) -> Option<&'static str> { match compression { - Compression::UNCOMPRESSED => None, - Compression::SNAPPY => Some("snappy"), - Compression::GZIP(_) => Some("gzip"), - Compression::LZO => Some("lzo"), - Compression::BROTLI(_) => Some("brotli"), - Compression::LZ4 => Some("lz4"), - Compression::ZSTD(_) => Some("zstd"), - Compression::LZ4_RAW => Some("lz4_raw"), + CompressionCodec::UNCOMPRESSED => None, + CompressionCodec::SNAPPY => Some("snappy"), + CompressionCodec::GZIP => Some("gzip"), + CompressionCodec::LZO => Some("lzo"), + CompressionCodec::BROTLI => Some("brotli"), + CompressionCodec::LZ4 => Some("lz4"), + CompressionCodec::ZSTD => Some("zstd"), + CompressionCodec::LZ4_RAW => Some("lz4_raw"), } } diff --git a/parquet/src/column/writer/mod.rs b/parquet/src/column/writer/mod.rs index 0c4e40b7ac59..06c00a6aa46e 100644 --- a/parquet/src/column/writer/mod.rs +++ b/parquet/src/column/writer/mod.rs @@ -1185,7 +1185,7 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { let data_page_offset = self.column_metrics.data_page_offset.unwrap_or(0) as i64; let mut builder = ColumnChunkMetaData::builder(self.descr.clone()) - .set_compression(self.codec) + .set_compression(self.codec.into()) .set_encodings_mask(EncodingMask::new_from_encodings(self.encodings.iter())) .set_page_encoding_stats(self.encoding_stats.clone()) .set_total_compressed_size(total_compressed_size) diff --git a/parquet/src/file/metadata/memory.rs b/parquet/src/file/metadata/memory.rs index 30c10e7f2293..8a5937c43c65 100644 --- a/parquet/src/file/metadata/memory.rs +++ b/parquet/src/file/metadata/memory.rs @@ -18,7 +18,7 @@ //! Memory calculations for [`ParquetMetadata::memory_size`] //! //! [`ParquetMetadata::memory_size`]: crate::file::metadata::ParquetMetaData::memory_size -use crate::basic::{BoundaryOrder, ColumnOrder, Compression, Encoding, PageType}; +use crate::basic::{BoundaryOrder, ColumnOrder, CompressionCodec, Encoding, PageType}; use crate::data_type::private::ParquetValueType; use crate::file::metadata::{ ColumnChunkMetaData, FileMetaData, KeyValue, PageEncodingStats, ParquetPageEncodingStats, @@ -206,7 +206,7 @@ impl HeapSize for SortingColumn { 0 // no heap allocations } } -impl HeapSize for Compression { +impl HeapSize for CompressionCodec { fn heap_size(&self) -> usize { 0 // no heap allocations } diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index 156385ddaa70..98e0a6a562df 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -95,9 +95,12 @@ pub(crate) mod reader; pub(crate) mod thrift; mod writer; -use crate::basic::{EncodingMask, PageType}; +use crate::basic::{ + BoundaryOrder, ColumnOrder, CompressionCodec, Encoding, EncodingMask, PageType, Type, +}; #[cfg(feature = "encryption")] use crate::encryption::decrypt::FileDecryptor; +use crate::errors::{ParquetError, Result}; #[cfg(feature = "encryption")] use crate::file::column_crypto_metadata::ColumnCryptoMetaData; pub(crate) use crate::file::metadata::memory::HeapSize; @@ -107,22 +110,15 @@ use crate::file::page_index::column_index::{ByteArrayColumnIndex, PrimitiveColum use crate::file::page_index::{column_index::ColumnIndexMetaData, offset_index::PageLocation}; use crate::file::statistics::Statistics; use crate::geospatial::statistics as geo_statistics; +use crate::parquet_thrift::{ + ElementType, FieldType, ReadThrift, ThriftCompactInputProtocol, ThriftCompactOutputProtocol, + WriteThrift, WriteThriftField, +}; use crate::schema::types::{ ColumnDescPtr, ColumnDescriptor, ColumnPath, SchemaDescPtr, SchemaDescriptor, Type as SchemaType, }; use crate::thrift_struct; -use crate::{ - basic::BoundaryOrder, - errors::{ParquetError, Result}, -}; -use crate::{ - basic::{ColumnOrder, Compression, Encoding, Type}, - parquet_thrift::{ - ElementType, FieldType, ReadThrift, ThriftCompactInputProtocol, - ThriftCompactOutputProtocol, WriteThrift, WriteThriftField, - }, -}; use crate::{ data_type::private::ParquetValueType, file::page_index::offset_index::OffsetIndexMetaData, }; @@ -814,7 +810,7 @@ pub struct ColumnChunkMetaData { file_path: Option, file_offset: i64, num_values: i64, - compression: Compression, + compression: CompressionCodec, total_compressed_size: i64, total_uncompressed_size: i64, data_page_offset: i64, @@ -1017,7 +1013,7 @@ impl ColumnChunkMetaData { } /// Compression for this column. - pub fn compression(&self) -> Compression { + pub fn compression(&self) -> CompressionCodec { self.compression } @@ -1234,7 +1230,7 @@ impl ColumnChunkMetaDataBuilder { file_path: None, file_offset: 0, num_values: 0, - compression: Compression::UNCOMPRESSED, + compression: CompressionCodec::UNCOMPRESSED, total_compressed_size: 0, total_uncompressed_size: 0, data_page_offset: 0, @@ -1286,7 +1282,7 @@ impl ColumnChunkMetaDataBuilder { } /// Sets compression. - pub fn set_compression(mut self, value: Compression) -> Self { + pub fn set_compression(mut self, value: CompressionCodec) -> Self { self.0.compression = value; self } @@ -1820,7 +1816,7 @@ mod tests { )) .set_file_path("file_path".to_owned()) .set_num_values(1000) - .set_compression(Compression::SNAPPY) + .set_compression(CompressionCodec::SNAPPY) .set_total_compressed_size(2000) .set_total_uncompressed_size(3000) .set_data_page_offset(4000) @@ -1860,7 +1856,7 @@ mod tests { )) .set_file_path("file_path".to_owned()) .set_num_values(1000) - .set_compression(Compression::SNAPPY) + .set_compression(CompressionCodec::SNAPPY) .set_total_compressed_size(2000) .set_total_uncompressed_size(3000) .set_data_page_offset(4000) @@ -1903,7 +1899,7 @@ mod tests { [Encoding::PLAIN, Encoding::RLE].iter(), )) .set_num_values(1000) - .set_compression(Compression::SNAPPY) + .set_compression(CompressionCodec::SNAPPY) .set_total_compressed_size(2000) .set_total_uncompressed_size(3000) .set_data_page_offset(4000) @@ -2034,9 +2030,9 @@ mod tests { .build(); #[cfg(not(feature = "encryption"))] - let base_expected_size = 2766; + let base_expected_size = 2734; #[cfg(feature = "encryption")] - let base_expected_size = 2934; + let base_expected_size = 2902; assert_eq!(parquet_meta.memory_size(), base_expected_size); @@ -2065,9 +2061,9 @@ mod tests { .build(); #[cfg(not(feature = "encryption"))] - let bigger_expected_size = 3192; + let bigger_expected_size = 3160; #[cfg(feature = "encryption")] - let bigger_expected_size = 3360; + let bigger_expected_size = 3328; // more set fields means more memory usage assert!(bigger_expected_size > base_expected_size); @@ -2114,7 +2110,7 @@ mod tests { .set_row_groups(row_group_meta.clone()) .build(); - let base_expected_size = 2058; + let base_expected_size = 2042; assert_eq!(parquet_meta_data.memory_size(), base_expected_size); let footer_key = "0123456789012345".as_bytes(); @@ -2140,7 +2136,7 @@ mod tests { .set_file_decryptor(Some(decryptor)) .build(); - let expected_size_with_decryptor = 3072; + let expected_size_with_decryptor = 3056; assert!(expected_size_with_decryptor > base_expected_size); assert_eq!( diff --git a/parquet/src/file/metadata/thrift/mod.rs b/parquet/src/file/metadata/thrift/mod.rs index ad8f6312c215..77d395253f85 100644 --- a/parquet/src/file/metadata/thrift/mod.rs +++ b/parquet/src/file/metadata/thrift/mod.rs @@ -35,8 +35,8 @@ use crate::file::{ }; use crate::{ basic::{ - ColumnOrder, Compression, ConvertedType, Encoding, EncodingMask, LogicalType, PageType, - Repetition, Type, + ColumnOrder, CompressionCodec, ConvertedType, Encoding, EncodingMask, LogicalType, + PageType, Repetition, Type, }, data_type::{ByteArray, FixedLenByteArray, Int96}, errors::{ParquetError, Result}, @@ -460,7 +460,7 @@ fn read_column_metadata<'a>( } // 3: path_in_schema is redundant 4 => { - column.compression = Compression::read_thrift(&mut *prot)?; + column.compression = CompressionCodec::read_thrift(&mut *prot)?; seen_mask |= COL_META_CODEC; } 5 => { diff --git a/parquet/src/file/serialized_reader.rs b/parquet/src/file/serialized_reader.rs index 254ccb779a4a..10a58b11c833 100644 --- a/parquet/src/file/serialized_reader.rs +++ b/parquet/src/file/serialized_reader.rs @@ -637,7 +637,7 @@ impl SerializedPageReader { page_locations: Option>, props: ReaderPropertiesPtr, ) -> Result { - let decompressor = create_codec(meta.compression(), props.codec_options())?; + let decompressor = create_codec(meta.compression().into(), props.codec_options())?; let (start, len) = meta.byte_range(); let state = match page_locations { diff --git a/parquet/src/file/writer.rs b/parquet/src/file/writer.rs index 7d69904451d3..48087d299713 100644 --- a/parquet/src/file/writer.rs +++ b/parquet/src/file/writer.rs @@ -1595,7 +1595,7 @@ mod tests { let desc = ColumnDescriptor::new(Arc::new(t), 0, 0, ColumnPath::new(vec![])); let meta = ColumnChunkMetaData::builder(Arc::new(desc)) - .set_compression(codec) + .set_compression(codec.into()) .set_total_compressed_size(reader.len() as i64) .set_num_values(total_num_values) .build() diff --git a/parquet/src/schema/printer.rs b/parquet/src/schema/printer.rs index 4ad3b6b93e00..2a77a9ba4361 100644 --- a/parquet/src/schema/printer.rs +++ b/parquet/src/schema/printer.rs @@ -180,7 +180,7 @@ fn print_column_chunk_metadata(out: &mut dyn io::Write, cc_metadata: &ColumnChun writeln!( out, "compression: {}", - cc_metadata.compression().codec_to_string() + cc_metadata.compression().to_string() ); writeln!( out, From fb4f54292e4f927867d8489734829883d71ecca9 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Thu, 30 Apr 2026 10:50:12 -0700 Subject: [PATCH 2/6] add test --- parquet/src/basic.rs | 63 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/parquet/src/basic.rs b/parquet/src/basic.rs index 133fd06cb54a..4d42761e8aec 100644 --- a/parquet/src/basic.rs +++ b/parquet/src/basic.rs @@ -2143,6 +2143,69 @@ mod tests { assert_eq!(Encoding::RLE_DICTIONARY.to_string(), "RLE_DICTIONARY"); } + #[test] + fn test_compression_conversion() { + assert_eq!( + CompressionCodec::from(Compression::UNCOMPRESSED), + CompressionCodec::UNCOMPRESSED + ); + assert_eq!( + CompressionCodec::from(Compression::SNAPPY), + CompressionCodec::SNAPPY + ); + assert_eq!( + CompressionCodec::from(Compression::GZIP(Default::default())), + CompressionCodec::GZIP + ); + assert_eq!( + CompressionCodec::from(Compression::LZO), + CompressionCodec::LZO + ); + assert_eq!( + CompressionCodec::from(Compression::BROTLI(Default::default())), + CompressionCodec::BROTLI + ); + assert_eq!( + CompressionCodec::from(Compression::LZ4), + CompressionCodec::LZ4 + ); + assert_eq!( + CompressionCodec::from(Compression::ZSTD(Default::default())), + CompressionCodec::ZSTD + ); + assert_eq!( + CompressionCodec::from(Compression::LZ4_RAW), + CompressionCodec::LZ4_RAW + ); + + assert_eq!( + Compression::from(CompressionCodec::UNCOMPRESSED), + Compression::UNCOMPRESSED + ); + assert_eq!( + Compression::from(CompressionCodec::SNAPPY), + Compression::SNAPPY + ); + assert_eq!( + Compression::from(CompressionCodec::GZIP), + Compression::GZIP(Default::default()) + ); + assert_eq!(Compression::from(CompressionCodec::LZO), Compression::LZO); + assert_eq!( + Compression::from(CompressionCodec::BROTLI), + Compression::BROTLI(Default::default()) + ); + assert_eq!(Compression::from(CompressionCodec::LZ4), Compression::LZ4); + assert_eq!( + Compression::from(CompressionCodec::ZSTD), + Compression::ZSTD(Default::default()) + ); + assert_eq!( + Compression::from(CompressionCodec::LZ4_RAW), + Compression::LZ4_RAW + ); + } + #[test] fn test_display_compression() { assert_eq!(Compression::UNCOMPRESSED.to_string(), "UNCOMPRESSED"); From 8368d1cc39d7498cc23e71d1753f4bbb90933fe4 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Thu, 30 Apr 2026 11:03:52 -0700 Subject: [PATCH 3/6] clippy --- parquet/benches/metadata.rs | 2 +- parquet/src/schema/printer.rs | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/parquet/benches/metadata.rs b/parquet/benches/metadata.rs index c9a6cf3b762c..0e7cd97b32ac 100644 --- a/parquet/benches/metadata.rs +++ b/parquet/benches/metadata.rs @@ -90,7 +90,7 @@ fn encoded_meta(is_nullable: bool, has_lists: bool) -> Vec { .map(|j| { ColumnChunkMetaData::builder(column_desc_ptrs[j].clone()) .set_encodings(vec![Encoding::PLAIN, Encoding::RLE_DICTIONARY]) - .set_compression(parquet::basic::Compression::UNCOMPRESSED) + .set_compression(parquet::basic::CompressionCodec::UNCOMPRESSED) .set_num_values(rng.random_range(1..1000000)) .set_total_compressed_size(rng.random_range(50000..5000000)) .set_data_page_offset(rng.random_range(4..2000000000)) diff --git a/parquet/src/schema/printer.rs b/parquet/src/schema/printer.rs index 2a77a9ba4361..770e411cd643 100644 --- a/parquet/src/schema/printer.rs +++ b/parquet/src/schema/printer.rs @@ -177,11 +177,7 @@ fn print_column_chunk_metadata(out: &mut dyn io::Write, cc_metadata: &ColumnChun writeln!(out, "file path: {file_path_str}"); writeln!(out, "file offset: {}", cc_metadata.file_offset()); writeln!(out, "num of values: {}", cc_metadata.num_values()); - writeln!( - out, - "compression: {}", - cc_metadata.compression().to_string() - ); + writeln!(out, "compression: {}", cc_metadata.compression()); writeln!( out, "total compressed size (in bytes): {}", From 924500ad3e5382b1bbb557c958e5447465c88d9c Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Fri, 1 May 2026 09:39:11 -0700 Subject: [PATCH 4/6] more docs and cleanup --- parquet/benches/metadata.rs | 2 +- parquet/src/bin/parquet-layout.rs | 2 +- parquet/src/column/writer/mod.rs | 2 +- parquet/src/file/metadata/mod.rs | 33 ++++++++++++++++++++------- parquet/src/file/serialized_reader.rs | 2 +- parquet/src/file/writer.rs | 4 ++-- parquet/src/schema/printer.rs | 2 +- 7 files changed, 32 insertions(+), 15 deletions(-) diff --git a/parquet/benches/metadata.rs b/parquet/benches/metadata.rs index 0e7cd97b32ac..fa1e08b62039 100644 --- a/parquet/benches/metadata.rs +++ b/parquet/benches/metadata.rs @@ -90,7 +90,7 @@ fn encoded_meta(is_nullable: bool, has_lists: bool) -> Vec { .map(|j| { ColumnChunkMetaData::builder(column_desc_ptrs[j].clone()) .set_encodings(vec![Encoding::PLAIN, Encoding::RLE_DICTIONARY]) - .set_compression(parquet::basic::CompressionCodec::UNCOMPRESSED) + .set_compression_codec(parquet::basic::CompressionCodec::UNCOMPRESSED) .set_num_values(rng.random_range(1..1000000)) .set_total_compressed_size(rng.random_range(50000..5000000)) .set_data_page_offset(rng.random_range(4..2000000000)) diff --git a/parquet/src/bin/parquet-layout.rs b/parquet/src/bin/parquet-layout.rs index 3a95c6178a0e..735a9665139a 100644 --- a/parquet/src/bin/parquet-layout.rs +++ b/parquet/src/bin/parquet-layout.rs @@ -118,7 +118,7 @@ fn do_layout(reader: &C) -> Result { .iter() .zip(schema.columns()) .map(|(column, column_schema)| { - let compression = compression(column.compression()); + let compression = compression(column.compression_codec()); let mut pages = vec![]; let mut start = column diff --git a/parquet/src/column/writer/mod.rs b/parquet/src/column/writer/mod.rs index 06c00a6aa46e..0c4e40b7ac59 100644 --- a/parquet/src/column/writer/mod.rs +++ b/parquet/src/column/writer/mod.rs @@ -1185,7 +1185,7 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { let data_page_offset = self.column_metrics.data_page_offset.unwrap_or(0) as i64; let mut builder = ColumnChunkMetaData::builder(self.descr.clone()) - .set_compression(self.codec.into()) + .set_compression(self.codec) .set_encodings_mask(EncodingMask::new_from_encodings(self.encodings.iter())) .set_page_encoding_stats(self.encoding_stats.clone()) .set_total_compressed_size(total_compressed_size) diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index 98e0a6a562df..f27e5aa09f8d 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -96,7 +96,8 @@ pub(crate) mod thrift; mod writer; use crate::basic::{ - BoundaryOrder, ColumnOrder, CompressionCodec, Encoding, EncodingMask, PageType, Type, + BoundaryOrder, ColumnOrder, Compression, CompressionCodec, Encoding, EncodingMask, PageType, + Type, }; #[cfg(feature = "encryption")] use crate::encryption::decrypt::FileDecryptor; @@ -1012,8 +1013,18 @@ impl ColumnChunkMetaData { self.num_values } - /// Compression for this column. - pub fn compression(&self) -> CompressionCodec { + /// [`Compression`] for this column. + /// + /// This is a default value suitable for passing to [`WriterPropertiesBuilder::set_compression`] + /// constructed from the `codec` field of the Parquet `ColumnMetaData` + /// + /// [`WriterPropertiesBuilder::set_compression`]: crate::file::properties::WriterPropertiesBuilder + pub fn compression(&self) -> Compression { + self.compression.into() + } + + /// Returns the compression codec used when writing this column. + pub fn compression_codec(&self) -> CompressionCodec { self.compression } @@ -1281,8 +1292,14 @@ impl ColumnChunkMetaDataBuilder { self } - /// Sets compression. - pub fn set_compression(mut self, value: CompressionCodec) -> Self { + /// Sets compression codec given a [`Compression`] configuration value. + pub fn set_compression(mut self, value: Compression) -> Self { + self.0.compression = value.into(); + self + } + + /// Sets compression codec. + pub fn set_compression_codec(mut self, value: CompressionCodec) -> Self { self.0.compression = value; self } @@ -1816,7 +1833,7 @@ mod tests { )) .set_file_path("file_path".to_owned()) .set_num_values(1000) - .set_compression(CompressionCodec::SNAPPY) + .set_compression_codec(CompressionCodec::SNAPPY) .set_total_compressed_size(2000) .set_total_uncompressed_size(3000) .set_data_page_offset(4000) @@ -1856,7 +1873,7 @@ mod tests { )) .set_file_path("file_path".to_owned()) .set_num_values(1000) - .set_compression(CompressionCodec::SNAPPY) + .set_compression_codec(CompressionCodec::SNAPPY) .set_total_compressed_size(2000) .set_total_uncompressed_size(3000) .set_data_page_offset(4000) @@ -1899,7 +1916,7 @@ mod tests { [Encoding::PLAIN, Encoding::RLE].iter(), )) .set_num_values(1000) - .set_compression(CompressionCodec::SNAPPY) + .set_compression_codec(CompressionCodec::SNAPPY) .set_total_compressed_size(2000) .set_total_uncompressed_size(3000) .set_data_page_offset(4000) diff --git a/parquet/src/file/serialized_reader.rs b/parquet/src/file/serialized_reader.rs index 10a58b11c833..254ccb779a4a 100644 --- a/parquet/src/file/serialized_reader.rs +++ b/parquet/src/file/serialized_reader.rs @@ -637,7 +637,7 @@ impl SerializedPageReader { page_locations: Option>, props: ReaderPropertiesPtr, ) -> Result { - let decompressor = create_codec(meta.compression().into(), props.codec_options())?; + let decompressor = create_codec(meta.compression(), props.codec_options())?; let (start, len) = meta.byte_range(); let state = match page_locations { diff --git a/parquet/src/file/writer.rs b/parquet/src/file/writer.rs index 48087d299713..ee2c007f2d2e 100644 --- a/parquet/src/file/writer.rs +++ b/parquet/src/file/writer.rs @@ -716,7 +716,7 @@ impl<'a, W: Write + Send> SerializedRowGroupWriter<'a, W> { let map_offset = |x| x - src_offset + write_offset as i64; let mut builder = ColumnChunkMetaData::builder(metadata.column_descr_ptr()) - .set_compression(metadata.compression()) + .set_compression_codec(metadata.compression_codec()) .set_encodings_mask(*metadata.encodings_mask()) .set_total_compressed_size(metadata.compressed_size()) .set_total_uncompressed_size(metadata.uncompressed_size()) @@ -1595,7 +1595,7 @@ mod tests { let desc = ColumnDescriptor::new(Arc::new(t), 0, 0, ColumnPath::new(vec![])); let meta = ColumnChunkMetaData::builder(Arc::new(desc)) - .set_compression(codec.into()) + .set_compression_codec(codec.into()) .set_total_compressed_size(reader.len() as i64) .set_num_values(total_num_values) .build() diff --git a/parquet/src/schema/printer.rs b/parquet/src/schema/printer.rs index 770e411cd643..68398005b6a5 100644 --- a/parquet/src/schema/printer.rs +++ b/parquet/src/schema/printer.rs @@ -177,7 +177,7 @@ fn print_column_chunk_metadata(out: &mut dyn io::Write, cc_metadata: &ColumnChun writeln!(out, "file path: {file_path_str}"); writeln!(out, "file offset: {}", cc_metadata.file_offset()); writeln!(out, "num of values: {}", cc_metadata.num_values()); - writeln!(out, "compression: {}", cc_metadata.compression()); + writeln!(out, "compression: {}", cc_metadata.compression_codec()); writeln!( out, "total compressed size (in bytes): {}", From 78f3509e8dcda3da3a14ad3ef7b487cd5afd3fd1 Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Fri, 1 May 2026 09:48:58 -0700 Subject: [PATCH 5/6] more wordsmithing --- parquet/src/file/metadata/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parquet/src/file/metadata/mod.rs b/parquet/src/file/metadata/mod.rs index f27e5aa09f8d..646438d2e979 100644 --- a/parquet/src/file/metadata/mod.rs +++ b/parquet/src/file/metadata/mod.rs @@ -1015,8 +1015,8 @@ impl ColumnChunkMetaData { /// [`Compression`] for this column. /// - /// This is a default value suitable for passing to [`WriterPropertiesBuilder::set_compression`] - /// constructed from the `codec` field of the Parquet `ColumnMetaData` + /// This is a default value suitable for passing to [`WriterPropertiesBuilder::set_compression`]. + /// It is constructed from the `codec` field of the Parquet `ColumnMetaData` /// /// [`WriterPropertiesBuilder::set_compression`]: crate::file::properties::WriterPropertiesBuilder pub fn compression(&self) -> Compression { From 234365868e6b565a0e67b8c82d69b5350e9af20a Mon Sep 17 00:00:00 2001 From: Ed Seidl Date: Fri, 1 May 2026 09:56:33 -0700 Subject: [PATCH 6/6] add note to revisit location of Compression enum --- parquet/src/basic.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/parquet/src/basic.rs b/parquet/src/basic.rs index 4d42761e8aec..cc7a16eca063 100644 --- a/parquet/src/basic.rs +++ b/parquet/src/basic.rs @@ -820,6 +820,11 @@ enum CompressionCodec { } ); +// NOTE: This enum likely belongs in file::properties now, but moving it there would be a +// breaking API change, that's probably not worth the pain. If a new codec is added to the +// Parquet specification, or any other breaking changes are made to this enum, this can be +// revisited. + /// Supported block compression algorithms. /// /// Block compression can yield non-trivial improvements to storage efficiency at the expense