From 86274fa180cbcb1d8b7411ea356ebfae78f9a8eb Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 23 Jun 2026 17:49:28 -0400 Subject: [PATCH 01/11] Make format versions data driven --- .../docs/File Format/implementationstatus.md | 2 +- content/en/docs/File Format/versions.md | 184 +----------------- .../features/compressions.yaml | 39 +++- data/implementations/features/encodings.yaml | 60 ++++-- .../features/format-features.yaml | 91 ++++++--- .../features/logical-types.yaml | 163 ++++++++++++---- .../features/physical-types.yaml | 42 +++- layouts/shortcodes/README.md | 15 ++ layouts/shortcodes/format-versions.html | 103 ++++++++++ layouts/shortcodes/implementation-status.html | 37 +++- 10 files changed, 463 insertions(+), 273 deletions(-) create mode 100644 layouts/shortcodes/README.md create mode 100644 layouts/shortcodes/format-versions.html diff --git a/content/en/docs/File Format/implementationstatus.md b/content/en/docs/File Format/implementationstatus.md index 47e7c22c..96cffb89 100644 --- a/content/en/docs/File Format/implementationstatus.md +++ b/content/en/docs/File Format/implementationstatus.md @@ -28,5 +28,5 @@ The value in each box means: * [duckdb](https://github.com/duckdb/duckdb) (C++) * [polars](https://github.com/pola-rs/polars) (Rust) - + {{< implementation-status >}} diff --git a/content/en/docs/File Format/versions.md b/content/en/docs/File Format/versions.md index 3f666d22..935cad02 100644 --- a/content/en/docs/File Format/versions.md +++ b/content/en/docs/File Format/versions.md @@ -72,42 +72,12 @@ the next parquet-format release. Forward incompatible features and the format version each became available in: -| Feature | Released in | Source | Notes | -| ------------------------------------------ | ----------------------------- | --- | ------------------------- | -| [BOOLEAN] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [INT32] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [INT64] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [INT96 (deprecated)] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [FLOAT] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [DOUBLE] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [BYTE_ARRAY] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [FIXED_LEN_BYTE_ARRAY] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [Data Page V1] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [Data Page V2] | [2.0.0] | [1.0.0..2.0.0] | | -| [PLAIN] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [PLAIN_DICTIONARY] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [RLE] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [BIT_PACKED (deprecated)] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [RLE_DICTIONARY] | [2.0.0] | [1.0.0..2.0.0] | | -| [DELTA_BINARY_PACKED] | [2.0.0] | [1.0.0..2.0.0] | | -| [DELTA_LENGTH_BYTE_ARRAY] | [2.0.0] | [1.0.0..2.0.0] | | -| [DELTA_BYTE_ARRAY] | [2.0.0] | [1.0.0..2.0.0] | | -| [BYTE_STREAM_SPLIT] | [2.8.0] | [2.7.0..2.8.0] | [Approved 2019-12-03] | -| [BYTE_STREAM_SPLIT
(Additional Types)] | [2.11.0] | [2.10.0..2.11.0] | [Approved 2024-03-18] | -| [UNCOMPRESSED] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [SNAPPY] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [GZIP] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [LZO] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [BROTLI] | [2.4.0] | [2.3.1..2.4.0] | | -| [LZ4 (deprecated)] | [2.4.0] | [2.3.1..2.4.0] | | -| [LZ4_RAW] | [2.9.0] | [2.8.0..2.9.0] | | -| [ZSTD] | [2.4.0] | [2.3.1..2.4.0] | | -| [Modular encryption] | [2.7.0] | [2.6.0..2.7.0] | [Approved 2019-01-16] | + +{{< format-versions table="forward_incompatible" >}} - -> **Note:** Files with an [encrypted footer] use different magic bytes (`PARE` +> **Note:** Files with an [encrypted footer](https://github.com/apache/parquet-format/blob/master/Encryption.md#54-encrypted-footer-mode) use different magic bytes (`PARE` > instead of `PAR1`), making it clear to readers they must support modular -> encryption to read the file; [plaintext footer] files use `PAR1` so legacy +> encryption to read the file; [plaintext footer](https://github.com/apache/parquet-format/blob/master/Encryption.md#55-plaintext-footer-mode) files use `PAR1` so legacy > readers can still read their unencrypted columns. ## Forward compatible additions @@ -115,147 +85,5 @@ Forward incompatible features and the format version each became available in: Older readers can read files that use these features but may not understand the new information. -| Feature | Released in | Source | Notes | -| ------------------------------------------- | ----------------------------- | --- |-----------------------------------------------------------| -| [xxHash-based bloom filters] | [2.7.0] | [2.6.0..2.7.0] | [Approved 2019-09-09] | -| [Bloom filter length] | [2.10.0] | [2.9.0..2.10.0] | | -| [Page index] | [2.4.0] | [2.3.1..2.4.0] | | -| [Page CRC32 checksum] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [Size statistics] | [2.10.0] | [2.9.0..2.10.0] | [Approved 2023-11-14] | -| [Geospatial statistics] | [2.11.0] | [2.10.0..2.11.0] | [Approved 2025-02-09] | -| [Binary protocol extensions] | [2.11.0] | [2.10.0..2.11.0] | [Approved 2024-09-06] | -| [IEEE 754 total order and NaN counts] | [2.13.0] | [2.12.0..2.13.0] | [Approved 2026-05-26] | -| [LogicalType union] | [2.4.0] | [2.3.1..2.4.0] | Supersedes `ConvertedType` enum
deprecated in [2.9.0] | -| [STRING (BYTE_ARRAY)] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [ENUM (BYTE_ARRAY)] | [2.0.0] | [1.0.0..2.0.0] | | -| [UUID (FIXED_LEN_BYTE_ARRAY(16))] | [2.6.0] | [2.5.0..2.6.0] | | -| [Signed and unsigned integer logical types (INT32, INT64)] | [2.2.0] | [2.1.0..2.2.0] | | -| [DECIMAL (INT32)] | [2.1.0] | [2.0.0..2.1.0] | | -| [DECIMAL (INT64)] | [2.1.0] | [2.0.0..2.1.0] | | -| [DECIMAL (BYTE_ARRAY)] | [2.1.0] | [2.0.0..2.1.0] | | -| [DECIMAL (FIXED_LEN_BYTE_ARRAY)] | [2.1.0] | [2.0.0..2.1.0] | | -| [FLOAT16 (FIXED_LEN_BYTE_ARRAY(2))] | [2.10.0] | [2.9.0..2.10.0] | [Approved 2023-10-13] | -| [DATE (INT32)] | [2.2.0] | [2.1.0..2.2.0] | | -| [TIME (INT32)] | [2.2.0] | [2.1.0..2.2.0] | | -| [TIME (INT64)] | [2.4.0] | [2.3.1..2.4.0] | | -| [TIMESTAMP (INT64)] | [2.2.0] | [2.1.0..2.2.0] | | -| [Nanosecond TIME/TIMESTAMP] | [2.6.0] | [2.5.0..2.6.0] | | -| [INTERVAL (FIXED_LEN_BYTE_ARRAY(12))] | [2.2.0] | [2.1.0..2.2.0] | | -| [JSON (BYTE_ARRAY)] | [2.2.0] | [2.1.0..2.2.0] | | -| [BSON (BYTE_ARRAY)] | [2.2.0] | [2.1.0..2.2.0] | | -| [VARIANT] | [2.12.0] | [2.11.0..2.12.0] | [Approved 2025-08-24] | -| [Variant shredding] | [2.12.0] | [2.11.0..2.12.0] | [Approved 2025-08-24] | -| [GEOMETRY (BYTE_ARRAY)] | [2.11.0] | [2.10.0..2.11.0] | [Approved 2025-02-09] | -| [GEOGRAPHY (BYTE_ARRAY)] | [2.11.0] | [2.10.0..2.11.0] | [Approved 2025-02-09] | -| [LIST] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [MAP] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [UNKNOWN (always null)] | [2.4.0] | [2.3.1..2.4.0] | | - -[PLAIN]: https://github.com/apache/parquet-format/blob/master/Encodings.md#plain-plain--0 -[PLAIN_DICTIONARY]: https://github.com/apache/parquet-format/blob/master/Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 -[RLE]: https://github.com/apache/parquet-format/blob/master/Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3 -[RLE_DICTIONARY]: https://github.com/apache/parquet-format/blob/master/Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 -[DELTA_BINARY_PACKED]: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-encoding-delta_binary_packed--5 -[DELTA_LENGTH_BYTE_ARRAY]: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-length-byte-array-delta_length_byte_array--6 -[DELTA_BYTE_ARRAY]: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-strings-delta_byte_array--7 -[BYTE_STREAM_SPLIT]: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 -[Modular encryption]: https://github.com/apache/parquet-format/blob/master/Encryption.md -[encrypted footer]: https://github.com/apache/parquet-format/blob/master/Encryption.md#54-encrypted-footer-mode -[plaintext footer]: https://github.com/apache/parquet-format/blob/master/Encryption.md#55-plaintext-footer-mode -[xxHash-based bloom filters]: https://github.com/apache/parquet-format/blob/master/BloomFilter.md -[Page index]: https://github.com/apache/parquet-format/blob/master/PageIndex.md -[FLOAT16 (FIXED_LEN_BYTE_ARRAY(2))]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#float16 -[VARIANT]: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md -[GEOMETRY (BYTE_ARRAY)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geometry -[GEOGRAPHY (BYTE_ARRAY)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geography - -[1.0.0]: https://github.com/apache/parquet-format/releases/tag/parquet-format-1.0.0 -[2.0.0]: https://github.com/apache/parquet-format/releases/tag/parquet-format-2.0.0 -[2.8.0]: https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.8.0 -[2.11.0]: https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.11.0 -[2.4.0]: https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.4.0 -[2.9.0]: https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.9.0 -[2.7.0]: https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.7.0 -[2.10.0]: https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.10.0 -[2.12.0]: https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.12.0 -[2.13.0]: https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.13.0 - -[Approved 2019-12-03]: https://lists.apache.org/thread/xs5qt2odm299pxgqb22mty2csc1so5yr -[Approved 2024-03-18]: https://lists.apache.org/thread/nlsj0ftxy7y4ov1678rgy5zc7dmogg6q -[Approved 2019-01-16]: https://lists.apache.org/thread/l8zcwnbrnhjh3w2k1lyb0v6ct5lnzr0h -[Approved 2019-09-09]: https://lists.apache.org/thread/ktdx1xp0d2gjfgkcvd29zxvt3cgg88bo -[Approved 2023-11-14]: https://lists.apache.org/thread/wgobz41mfldbhqpg9q4mdwypghg2cxg2 -[Approved 2023-10-13]: https://lists.apache.org/thread/gyvqcx9ssxkjlrwogqwy7n4z6ofdm871 -[Approved 2025-08-24]: https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7 -[Approved 2025-02-09]: https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo - -[1.0.0..2.0.0]: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 -[2.7.0..2.8.0]: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.7.0...apache-parquet-format-2.8.0 -[2.10.0..2.11.0]: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 -[2.3.1..2.4.0]: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 -[2.8.0..2.9.0]: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.8.0...apache-parquet-format-2.9.0 -[2.6.0..2.7.0]: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.6.0...apache-parquet-format-2.7.0 -[2.9.0..2.10.0]: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.9.0...apache-parquet-format-2.10.0 -[2.11.0..2.12.0]: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.11.0...apache-parquet-format-2.12.0 -[2.12.0..2.13.0]: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.12.0...apache-parquet-format-2.13.0 - -[2.1.0]: https://github.com/apache/parquet-format/releases/tag/parquet-format-2.1.0 -[2.2.0]: https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.2.0 -[2.6.0]: https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.6.0 -[2.0.0..2.1.0]: https://github.com/apache/parquet-format/compare/parquet-format-2.0.0...parquet-format-2.1.0 -[2.1.0..2.2.0]: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 -[2.5.0..2.6.0]: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.5.0...apache-parquet-format-2.6.0 - -[tree-1.0.0]: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - -[Variant shredding]: https://github.com/apache/parquet-format/blob/master/VariantShredding.md -[Geospatial statistics]: https://github.com/apache/parquet-format/blob/master/Geospatial.md#statistics -[Binary protocol extensions]: https://github.com/apache/parquet-format/blob/master/BinaryProtocolExtensions.md -[Approved 2024-09-06]: https://lists.apache.org/thread/x3472kldrq5kjnld9ztj1jozz25f40hg -[Approved 2026-05-26]: https://lists.apache.org/thread/h0k0hqo0sojqphnbnrkp8b0gmwdzq9on - -[Bloom filter length]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L933 -[Page CRC32 checksum]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L829 -[Size statistics]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L202 -[LogicalType union]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L471 -[Nanosecond TIME/TIMESTAMP]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L352 -[STRING (BYTE_ARRAY)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#string -[ENUM (BYTE_ARRAY)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#enum -[UUID (FIXED_LEN_BYTE_ARRAY(16))]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#uuid -[DECIMAL (INT32)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal -[DECIMAL (INT64)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal -[DECIMAL (BYTE_ARRAY)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal -[DECIMAL (FIXED_LEN_BYTE_ARRAY)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal -[DATE (INT32)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#date -[TIME (INT32)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time -[TIME (INT64)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time -[TIMESTAMP (INT64)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#timestamp -[INTERVAL (FIXED_LEN_BYTE_ARRAY(12))]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#interval -[JSON (BYTE_ARRAY)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#json -[BSON (BYTE_ARRAY)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#bson -[LIST]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists -[MAP]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#maps -[UNKNOWN (always null)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#unknown-always-null -[Signed and unsigned integer logical types (INT32, INT64)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#signed-integers -[IEEE 754 total order and NaN counts]: https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift#L1061 - -[BOOLEAN]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L33 -[INT32]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L34 -[INT64]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L35 -[INT96 (deprecated)]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L36 -[FLOAT]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L37 -[DOUBLE]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L38 -[BYTE_ARRAY]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L39 -[FIXED_LEN_BYTE_ARRAY]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L40 -[Data Page V1]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L671 -[Data Page V2]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L724 -[BIT_PACKED (deprecated)]: https://github.com/apache/parquet-format/blob/master/Encodings.md#bit-packed-deprecated-bit_packed--4 -[BYTE_STREAM_SPLIT
(Additional Types)]: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 -[UNCOMPRESSED]: https://github.com/apache/parquet-format/blob/master/Compression.md#uncompressed -[SNAPPY]: https://github.com/apache/parquet-format/blob/master/Compression.md#snappy -[GZIP]: https://github.com/apache/parquet-format/blob/master/Compression.md#gzip -[LZO]: https://github.com/apache/parquet-format/blob/master/Compression.md#lzo -[BROTLI]: https://github.com/apache/parquet-format/blob/master/Compression.md#brotli -[LZ4 (deprecated)]: https://github.com/apache/parquet-format/blob/master/Compression.md#lz4 -[LZ4_RAW]: https://github.com/apache/parquet-format/blob/master/Compression.md#lz4_raw -[ZSTD]: https://github.com/apache/parquet-format/blob/master/Compression.md#zstd + +{{< format-versions table="forward_compatible" >}} diff --git a/data/implementations/features/compressions.yaml b/data/implementations/features/compressions.yaml index fec73a0c..2f523ac5 100644 --- a/data/implementations/features/compressions.yaml +++ b/data/implementations/features/compressions.yaml @@ -2,24 +2,49 @@ category_id: compressions features: - id: compression-uncompressed display_name: UNCOMPRESSED - + compatibility: forward_incompatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#uncompressed + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - id: compression-brotli display_name: BROTLI - + compatibility: forward_incompatible + released_in: 2.4.0 + feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#brotli + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 - id: compression-gzip display_name: GZIP - + compatibility: forward_incompatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#gzip + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - id: compression-lz4-deprecated display_name: LZ4 (deprecated) - + compatibility: forward_incompatible + released_in: 2.4.0 + feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lz4 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 - id: compression-lz4-raw display_name: LZ4_RAW - + compatibility: forward_incompatible + released_in: 2.9.0 + feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lz4_raw + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.8.0...apache-parquet-format-2.9.0 - id: compression-lzo display_name: LZO - + compatibility: forward_incompatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lzo + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - id: compression-snappy display_name: SNAPPY - + compatibility: forward_incompatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#snappy + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - id: compression-zstd display_name: ZSTD + compatibility: forward_incompatible + released_in: 2.4.0 + feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#zstd + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 diff --git a/data/implementations/features/encodings.yaml b/data/implementations/features/encodings.yaml index 03614d9b..ac31a5db 100644 --- a/data/implementations/features/encodings.yaml +++ b/data/implementations/features/encodings.yaml @@ -2,37 +2,67 @@ category_id: encodings features: - id: encoding-plain display_name: PLAIN - + compatibility: forward_incompatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#plain-plain--0 + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - id: encoding-plain-dictionary display_name: PLAIN_DICTIONARY - + compatibility: forward_incompatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - id: encoding-rle-dictionary display_name: RLE_DICTIONARY - + compatibility: forward_incompatible + released_in: 2.0.0 + feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 + source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 - id: encoding-rle display_name: RLE - + compatibility: forward_incompatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3 + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - id: encoding-bit-packed display_name: BIT_PACKED (deprecated) - + compatibility: forward_incompatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#bit-packed-deprecated-bit_packed--4 + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - id: encoding-delta-binary-packed display_name: DELTA_BINARY_PACKED - + compatibility: forward_incompatible + released_in: 2.0.0 + feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-encoding-delta_binary_packed--5 + source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 - id: encoding-delta-length-byte-array display_name: DELTA_LENGTH_BYTE_ARRAY - + compatibility: forward_incompatible + released_in: 2.0.0 + feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-length-byte-array-delta_length_byte_array--6 + source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 - id: encoding-delta-byte-array display_name: DELTA_BYTE_ARRAY - + compatibility: forward_incompatible + released_in: 2.0.0 + feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-strings-delta_byte_array--7 + source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 - id: encoding-byte-stream-split display_name: BYTE_STREAM_SPLIT - format_version: - version: "2.8.0" - date: "2020-01-13" - + compatibility: forward_incompatible + released_in: 2.8.0 + release_date: '2020-01-13' + feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.7.0...apache-parquet-format-2.8.0 + version_note: "[Approved 2019-12-03](https://lists.apache.org/thread/xs5qt2odm299pxgqb22mty2csc1so5yr)" - id: encoding-byte-stream-split-extended display_name: BYTE_STREAM_SPLIT (Additional Types) spec_url: https://github.com/apache/parquet-format/commit/e517ac4dbe08d518eb5c2e58576d4c711973db94 - format_version: - version: "2.11.0" - date: "2024-03-21" + version_display_name: BYTE_STREAM_SPLIT
(Additional Types) + compatibility: forward_incompatible + released_in: 2.11.0 + release_date: '2024-03-21' + feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 + version_note: "[Approved 2024-03-18](https://lists.apache.org/thread/nlsj0ftxy7y4ov1678rgy5zc7dmogg6q)" diff --git a/data/implementations/features/format-features.yaml b/data/implementations/features/format-features.yaml index 5a2dbfa4..b25e3438 100644 --- a/data/implementations/features/format-features.yaml +++ b/data/implementations/features/format-features.yaml @@ -3,41 +3,88 @@ features: - id: format-bloom-filters display_name: xxHash-based bloom filters spec_url: https://github.com/apache/parquet-format/blob/master/BloomFilter.md - format_version: - version: "2.7.0" - date: "2019-09-25" - + compatibility: forward_compatible + released_in: 2.7.0 + release_date: '2019-09-25' + feature_url: https://github.com/apache/parquet-format/blob/master/BloomFilter.md + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.6.0...apache-parquet-format-2.7.0 + version_note: "[Approved 2019-09-09](https://lists.apache.org/thread/ktdx1xp0d2gjfgkcvd29zxvt3cgg88bo)" - id: format-bloom-filter-length display_name: Bloom filter length - note: "In parquet.thrift: ColumnMetaData->bloom_filter_length" - + note: 'In parquet.thrift: ColumnMetaData->bloom_filter_length' + compatibility: forward_compatible + released_in: 2.10.0 + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L933 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.9.0...apache-parquet-format-2.10.0 - id: format-stats-min-max display_name: Statistics min_value, max_value - - id: format-page-index display_name: Page index spec_url: https://github.com/apache/parquet-format/blob/master/PageIndex.md - format_version: - version: "2.4.0" - date: "2018-05-04" - + compatibility: forward_compatible + released_in: 2.4.0 + release_date: '2018-05-04' + feature_url: https://github.com/apache/parquet-format/blob/master/PageIndex.md + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 - id: format-page-crc32 display_name: Page CRC32 checksum - + compatibility: forward_compatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L829 + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - id: format-modular-encryption display_name: Modular encryption spec_url: https://github.com/apache/parquet-format/blob/master/Encryption.md - format_version: - version: "2.7.0" - date: "2019-09-25" - + compatibility: forward_incompatible + released_in: 2.7.0 + release_date: '2019-09-25' + feature_url: https://github.com/apache/parquet-format/blob/master/Encryption.md + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.6.0...apache-parquet-format-2.7.0 + version_note: "[Approved 2019-01-16](https://lists.apache.org/thread/l8zcwnbrnhjh3w2k1lyb0v6ct5lnzr0h)" - id: format-size-statistics display_name: Size statistics - note: "In parquet.thrift: ColumnMetaData->size_statistics" - format_version: - version: "2.10.0" - date: "2023-11-20" - + note: 'In parquet.thrift: ColumnMetaData->size_statistics' + compatibility: forward_compatible + released_in: 2.10.0 + release_date: '2023-11-20' + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L202 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.9.0...apache-parquet-format-2.10.0 + version_note: "[Approved 2023-11-14](https://lists.apache.org/thread/wgobz41mfldbhqpg9q4mdwypghg2cxg2)" - id: format-data-page-v2 display_name: Data Page V2 - note: "In parquet.thrift: DataPageHeaderV2" + note: 'In parquet.thrift: DataPageHeaderV2' + compatibility: forward_incompatible + released_in: 2.0.0 + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L724 + source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 + - id: format-data-page-v1 + display_name: Data Page V1 + implementation_status: false + compatibility: forward_incompatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L671 + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + - id: format-geospatial-statistics + display_name: Geospatial statistics + implementation_status: false + compatibility: forward_compatible + released_in: 2.11.0 + feature_url: https://github.com/apache/parquet-format/blob/master/Geospatial.md#statistics + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 + version_note: "[Approved 2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" + - id: format-binary-protocol-extensions + display_name: Binary protocol extensions + implementation_status: false + compatibility: forward_compatible + released_in: 2.11.0 + feature_url: https://github.com/apache/parquet-format/blob/master/BinaryProtocolExtensions.md + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 + version_note: "[Approved 2024-09-06](https://lists.apache.org/thread/x3472kldrq5kjnld9ztj1jozz25f40hg)" + - id: format-ieee-754-total-order-and-nan-counts + display_name: IEEE 754 total order and NaN counts + implementation_status: false + compatibility: forward_compatible + released_in: 2.13.0 + feature_url: https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift#L1061 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.12.0...apache-parquet-format-2.13.0 + version_note: "[Approved 2026-05-26](https://lists.apache.org/thread/h0k0hqo0sojqphnbnrkp8b0gmwdzq9on)" diff --git a/data/implementations/features/logical-types.yaml b/data/implementations/features/logical-types.yaml index 5d83b0bf..fced412c 100644 --- a/data/implementations/features/logical-types.yaml +++ b/data/implementations/features/logical-types.yaml @@ -2,81 +2,178 @@ category_id: logical-types features: - id: logical-string display_name: STRING - + version_display_name: STRING (BYTE_ARRAY) + compatibility: forward_compatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#string + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - id: logical-enum display_name: ENUM - + version_display_name: ENUM (BYTE_ARRAY) + compatibility: forward_compatible + released_in: 2.0.0 + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#enum + source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 - id: logical-uuid display_name: UUID - + version_display_name: UUID (FIXED_LEN_BYTE_ARRAY(16)) + compatibility: forward_compatible + released_in: 2.6.0 + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#uuid + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.5.0...apache-parquet-format-2.6.0 - id: logical-int-types display_name: 8, 16, 32, 64 bit signed and unsigned INT - + version_display_name: Signed and unsigned integer logical types (INT32, INT64) + compatibility: forward_compatible + released_in: 2.2.0 + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#signed-integers + source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 - id: logical-decimal-int32 display_name: DECIMAL (INT32) - + compatibility: forward_compatible + released_in: 2.1.0 + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal + source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.0.0...parquet-format-2.1.0 - id: logical-decimal-int64 display_name: DECIMAL (INT64) - + compatibility: forward_compatible + released_in: 2.1.0 + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal + source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.0.0...parquet-format-2.1.0 - id: logical-decimal-byte-array display_name: DECIMAL (BYTE_ARRAY) - + compatibility: forward_compatible + released_in: 2.1.0 + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal + source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.0.0...parquet-format-2.1.0 - id: logical-decimal-fixed-len-byte-array display_name: DECIMAL (FIXED_LEN_BYTE_ARRAY) - + compatibility: forward_compatible + released_in: 2.1.0 + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal + source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.0.0...parquet-format-2.1.0 - id: logical-float16 display_name: FLOAT16 - format_version: - version: "2.10.0" - date: "2023-11-20" - + version_display_name: FLOAT16 (FIXED_LEN_BYTE_ARRAY(2)) + compatibility: forward_compatible + released_in: 2.10.0 + release_date: '2023-11-20' + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#float16 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.9.0...apache-parquet-format-2.10.0 + version_note: "[Approved 2023-10-13](https://lists.apache.org/thread/gyvqcx9ssxkjlrwogqwy7n4z6ofdm871)" - id: logical-date display_name: DATE - + version_display_name: DATE (INT32) + compatibility: forward_compatible + released_in: 2.2.0 + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#date + source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 - id: logical-time-int32 display_name: TIME (INT32) - + compatibility: forward_compatible + released_in: 2.2.0 + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time + source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 - id: logical-time-int64 display_name: TIME (INT64) - + compatibility: forward_compatible + released_in: 2.4.0 + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 - id: logical-timestamp-int64 display_name: TIMESTAMP (INT64) - + compatibility: forward_compatible + released_in: 2.2.0 + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#timestamp + source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 - id: logical-interval display_name: INTERVAL - + version_display_name: INTERVAL (FIXED_LEN_BYTE_ARRAY(12)) + compatibility: forward_compatible + released_in: 2.2.0 + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#interval + source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 - id: logical-json display_name: JSON - + version_display_name: JSON (BYTE_ARRAY) + compatibility: forward_compatible + released_in: 2.2.0 + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#json + source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 - id: logical-bson display_name: BSON - + version_display_name: BSON (BYTE_ARRAY) + compatibility: forward_compatible + released_in: 2.2.0 + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#bson + source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 - id: logical-variant display_name: VARIANT spec_url: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md - format_version: - version: "2.11.0" - date: "2025-03-23" - + compatibility: forward_compatible + released_in: 2.12.0 + release_date: '2025-03-23' + feature_url: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.11.0...apache-parquet-format-2.12.0 + version_note: "[Approved 2025-08-24](https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7)" - id: logical-geometry display_name: GEOMETRY spec_url: https://github.com/apache/parquet-format/blob/master/Geospatial.md#logical-types - format_version: - version: "2.11.0" - date: "2025-03-23" - + version_display_name: GEOMETRY (BYTE_ARRAY) + compatibility: forward_compatible + released_in: 2.11.0 + release_date: '2025-03-23' + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geometry + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 + version_note: "[Approved 2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: logical-geography display_name: GEOGRAPHY spec_url: https://github.com/apache/parquet-format/blob/master/Geospatial.md#logical-types - format_version: - version: "2.11.0" - date: "2025-03-23" - + version_display_name: GEOGRAPHY (BYTE_ARRAY) + compatibility: forward_compatible + released_in: 2.11.0 + release_date: '2025-03-23' + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geography + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 + version_note: "[Approved 2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: logical-list display_name: LIST - + compatibility: forward_compatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - id: logical-map display_name: MAP - + compatibility: forward_compatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#maps + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - id: logical-unknown display_name: UNKNOWN (always null) + compatibility: forward_compatible + released_in: 2.4.0 + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#unknown-always-null + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 + - id: logical-type-union + display_name: LogicalType union + implementation_status: false + compatibility: forward_compatible + released_in: 2.4.0 + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L471 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 + version_note: Supersedes `ConvertedType` enum
deprecated in [2.9.0](https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.9.0) + - id: logical-nanosecond-time-timestamp + display_name: Nanosecond TIME/TIMESTAMP + implementation_status: false + compatibility: forward_compatible + released_in: 2.6.0 + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L352 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.5.0...apache-parquet-format-2.6.0 + - id: logical-variant-shredding + display_name: Variant shredding + implementation_status: false + compatibility: forward_compatible + released_in: 2.12.0 + feature_url: https://github.com/apache/parquet-format/blob/master/VariantShredding.md + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.11.0...apache-parquet-format-2.12.0 + version_note: "[Approved 2025-08-24](https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7)" diff --git a/data/implementations/features/physical-types.yaml b/data/implementations/features/physical-types.yaml index 38d18711..c445fe13 100644 --- a/data/implementations/features/physical-types.yaml +++ b/data/implementations/features/physical-types.yaml @@ -2,25 +2,51 @@ category_id: physical-types features: - id: physical-boolean display_name: BOOLEAN - + compatibility: forward_incompatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L33 + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - id: physical-int32 display_name: INT32 - + compatibility: forward_incompatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L34 + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - id: physical-int64 display_name: INT64 - + compatibility: forward_incompatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L35 + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - id: physical-int96 display_name: INT96 - note: "This type is deprecated, but as of 2024 it's common in currently produced parquet files" - + note: This type is deprecated, but as of 2024 it's common in currently produced parquet files + version_display_name: INT96 (deprecated) + compatibility: forward_incompatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L36 + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - id: physical-float display_name: FLOAT - + compatibility: forward_incompatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L37 + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - id: physical-double display_name: DOUBLE - + compatibility: forward_incompatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L38 + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - id: physical-byte-array display_name: BYTE_ARRAY - + compatibility: forward_incompatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L39 + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - id: physical-fixed-len-byte-array display_name: FIXED_LEN_BYTE_ARRAY + compatibility: forward_incompatible + released_in: 1.0.0 + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L40 + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 diff --git a/layouts/shortcodes/README.md b/layouts/shortcodes/README.md new file mode 100644 index 00000000..da45bd56 --- /dev/null +++ b/layouts/shortcodes/README.md @@ -0,0 +1,15 @@ +# Hugo shortcodes + +Shortcodes are Hugo templates called from Markdown content; +see the [Hugo shortcode documentation] for details. + +This site uses shortcodes to dynamically generate content from data stored in +the `data/` directory. + +For example the [format-versions.html] shortcode is used in the [versions.md] +page to generate the format version tables. The page calls it with Hugo's +shortcode syntax and passes a `table` parameter such as `forward_incompatible`. + +[format-versions.html]: format-versions.html +[versions.md]: ../../content/en/docs/File%20Format/versions.md +[Hugo shortcode documentation]: https://gohugo.io/content-management/shortcodes/#custom diff --git a/layouts/shortcodes/format-versions.html b/layouts/shortcodes/format-versions.html new file mode 100644 index 00000000..1bebd453 --- /dev/null +++ b/layouts/shortcodes/format-versions.html @@ -0,0 +1,103 @@ +{{- /* + Render Parquet format version feature tables from data/implementations/features/*.yaml. + See layouts/shortcodes/README.md for details. + Usage: {{< format-versions table="forward_incompatible" >}} +*/ -}} +{{- $tableName := .Get "table" -}} +{{- $validTables := slice "forward_incompatible" "forward_compatible" -}} +{{- if not (in $validTables $tableName) -}} + {{- errorf "unknown format versions table %q" $tableName -}} +{{- end -}} + +{{- $features := slice -}} +{{- range site.Data.implementations.categories -}} + {{- $featuresData := index site.Data.implementations.features .id -}} + {{- if $featuresData -}} + {{- range $featuresData.features -}} + {{- if eq .compatibility $tableName -}} + {{- $releasedIn := printf "%v" .released_in -}} + {{- $versionParts := split $releasedIn "." -}} + {{- $major := int (index $versionParts 0) -}} + {{- $minor := 0 -}} + {{- $patch := 0 -}} + {{- if ge (len $versionParts) 2 -}} + {{- $minor = int (index $versionParts 1) -}} + {{- end -}} + {{- if ge (len $versionParts) 3 -}} + {{- $patch = int (index $versionParts 2) -}} + {{- end -}} + {{- $sortKey := printf "%04d.%04d.%04d.%s" $major $minor $patch .id -}} + {{- $features = $features | append (dict "sort_key" $sortKey "feature" .) -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- $features = sort $features "sort_key" "desc" -}} + + + + + + + + + + + + {{- range $features }} + {{- $feature := .feature -}} + {{- $displayName := $feature.display_name -}} + {{- if $feature.version_display_name -}} + {{- $displayName = $feature.version_display_name -}} + {{- end -}} + {{- $featureUrl := $feature.feature_url -}} + {{- if and (not $featureUrl) $feature.spec_url -}} + {{- $featureUrl = $feature.spec_url -}} + {{- end -}} + {{- $releasedIn := printf "%v" $feature.released_in -}} + {{- $legacyReleaseTags := slice "1.0.0" "2.0.0" "2.1.0" -}} + {{- $releaseTag := printf "apache-parquet-format-%s" $releasedIn -}} + {{- if in $legacyReleaseTags $releasedIn -}} + {{- $releaseTag = printf "parquet-format-%s" $releasedIn -}} + {{- end -}} + {{- $releaseUrl := printf "https://github.com/apache/parquet-format/releases/tag/%s" $releaseTag -}} + {{- $sourceUrl := $feature.source_url -}} + {{- $sourceLabel := "" -}} + {{- if in $sourceUrl "/compare/" -}} + {{- $compareRange := replaceRE "^.*/compare/" "" $sourceUrl -}} + {{- $compareRange = replace $compareRange "apache-parquet-format-" "" -}} + {{- $compareRange = replace $compareRange "parquet-format-" "" -}} + {{- $sourceLabel = replace $compareRange "..." ".." -}} + {{- else -}} + {{- $sourceLabel = path.Base $sourceUrl -}} + {{- $sourceLabel = replace $sourceLabel "apache-parquet-format-" "" -}} + {{- $sourceLabel = replace $sourceLabel "parquet-format-" "" -}} + {{- end -}} + + + + + + + {{- end }} + +
FeatureReleased inSourceApproved
+ {{- if $featureUrl -}} + {{ $displayName | safeHTML }} + {{- else -}} + {{ $displayName | safeHTML }} + {{- end -}} + + {{ $releasedIn }} + + {{ $sourceLabel }} + + {{- $approvalNote := $feature.version_note -}} + {{- if hasPrefix $approvalNote "[Approved " -}} + {{- $approvalDate := replaceRE `^\[Approved ([0-9-]+)\]\(([^)]+)\)$` "$1" $approvalNote -}} + {{- $approvalUrl := replaceRE `^\[Approved ([0-9-]+)\]\(([^)]+)\)$` "$2" $approvalNote -}} + {{- if ne $approvalDate $approvalNote -}} + {{ $approvalDate }} + {{- end -}} + {{- end -}} +
diff --git a/layouts/shortcodes/implementation-status.html b/layouts/shortcodes/implementation-status.html index 0008c443..4dcc3ca6 100644 --- a/layouts/shortcodes/implementation-status.html +++ b/layouts/shortcodes/implementation-status.html @@ -1,5 +1,6 @@ {{- /* - Render implementation status tables from structured data in `data/implementation` + Render implementation status tables from structured data in `data/implementations` + See layouts/shortcodes/README.md for details. Usage: {{< implementation-status >}} */ -}} @@ -45,6 +46,8 @@

{{ $category.name }} {{- range $features -}} + {{- /* Include rows by default; skip rows explicitly marked implementation_status: false. */ -}} + {{- if ne .implementation_status false -}} {{- $feature := . -}} {{- $featureId := .id -}} @@ -56,10 +59,10 @@

{{ $category.name }} {{ .display_name }} {{- end -}} - {{- /* Render format version inline if present */ -}} - {{- if .format_version -}} + {{- /* Render format release year inline if present. */ -}} + {{- if .release_date -}} {{- $year := "" -}} - {{- $dateStr := .format_version.date -}} + {{- $dateStr := .release_date -}} {{- if hasPrefix $dateStr "2025" -}} {{- $year = "2025" -}} {{- else if hasPrefix $dateStr "2024" -}} @@ -75,7 +78,13 @@

{{ $category.name }} {{- else -}} {{- $year = substr $dateStr 0 4 -}} {{- end -}} - {{- $releaseUrl := printf "https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-%s" .format_version.version -}} + {{- $releasedIn := printf "%v" .released_in -}} + {{- $legacyReleaseTags := slice "1.0.0" "2.0.0" "2.1.0" -}} + {{- $releaseTag := printf "apache-parquet-format-%s" $releasedIn -}} + {{- if in $legacyReleaseTags $releasedIn -}} + {{- $releaseTag = printf "parquet-format-%s" $releasedIn -}} + {{- end -}} + {{- $releaseUrl := printf "https://github.com/apache/parquet-format/releases/tag/%s" $releaseTag -}} ({{ $year }}) {{- end -}} @@ -236,6 +245,7 @@

{{ $category.name }} {{- end -}} + {{- end -}} {{- end -}} @@ -255,7 +265,7 @@

Minimum Version fo

Note: This data was originally collected in December 2025, and not all data was backfilled. It is likely older releases of each engine support reading all features for 2023 and before. As volunteers have time they are invited to add more granular details on releases. Generally, versions are expected to be accurate for any year 2025 and after.

-{{- /* Collect all features with format_version and group by year */ -}} +{{- /* Collect all features with release_date and group by year */ -}} {{- $featuresByYear := dict "2023" (slice) "2024" (slice) "2025" (slice) -}} {{- $excludedFeatures := slice "encoding-bit-packed" "compression-lz4-deprecated" "compression-lzo" "logical-enum" "logical-uuid" "logical-bson" "logical-json" "logical-interval" -}} {{- $includedCategories := slice "compressions" "encodings" "physical-types" "logical-types" -}} @@ -267,12 +277,14 @@

Minimum Version fo {{- $featuresData := index site.Data.implementations.features $categoryId -}} {{- if $featuresData -}} {{- range $featuresData.features -}} + {{- /* Include rows by default; skip rows explicitly marked implementation_status: false. */ -}} + {{- if ne .implementation_status false -}} {{- $feature := . -}} {{- /* Skip excluded features */ -}} {{- if not (in $excludedFeatures .id) -}} {{- $year := "" -}} - {{- if .format_version -}} - {{- $dateStr := .format_version.date -}} + {{- if .release_date -}} + {{- $dateStr := .release_date -}} {{- if hasPrefix $dateStr "2025" -}} {{- $year = "2025" -}} {{- else if hasPrefix $dateStr "2024" -}} @@ -281,13 +293,14 @@

Minimum Version fo {{- $year = "2023" -}} {{- end -}} {{- else -}} - {{- /* Features without format_version are assumed pre-2023 */ -}} + {{- /* Features without release_date are assumed pre-2023 */ -}} {{- $year = "2023" -}} {{- end -}} {{- $currentList := index $featuresByYear $year -}} {{- $featuresByYear = merge $featuresByYear (dict $year ($currentList | append $feature)) -}} {{- end -}} + {{- end -}} {{- end -}} {{- end -}} {{- end -}} @@ -301,9 +314,12 @@

Minimum Version fo {{- $featuresData := index site.Data.implementations.features $categoryId -}} {{- if $featuresData -}} {{- range $featuresData.features -}} + {{- /* Include rows by default; skip rows explicitly marked implementation_status: false. */ -}} + {{- if ne .implementation_status false -}} {{- if in $excludedFeatures .id -}} {{- $excludedFeatureNames = $excludedFeatureNames | append .display_name -}} {{- end -}} + {{- end -}} {{- end -}} {{- end -}} {{- end -}} @@ -353,6 +369,8 @@

Minimum Version fo {{- /* Find minimum version that supports all features in this year */ -}} {{- if $supportData -}} {{- range $features -}} + {{- /* Include rows by default; skip rows explicitly marked implementation_status: false. */ -}} + {{- if ne .implementation_status false -}} {{- $featureId := .id -}} {{- $support := index $supportData.support $featureId -}} @@ -398,6 +416,7 @@

Minimum Version fo {{- /* Feature is not supported or only has write support */ -}} {{- $missingFeatures = $missingFeatures | append .display_name -}} {{- end -}} + {{- end -}} {{- end -}} {{- /* Find release notes URL for minimum version */ -}} From 708f0125ef8e533c63c6296091f43489cbebc339 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 24 Jun 2026 12:16:25 -0400 Subject: [PATCH 02/11] Use reference links for footer note --- content/en/docs/File Format/versions.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/content/en/docs/File Format/versions.md b/content/en/docs/File Format/versions.md index 935cad02..00b7f0df 100644 --- a/content/en/docs/File Format/versions.md +++ b/content/en/docs/File Format/versions.md @@ -67,6 +67,8 @@ the next parquet-format release. [contributing guide]: https://github.com/apache/parquet-format/blob/master/CONTRIBUTING.md#additionschanges-to-the-format [note in parquet.thrift]: https://github.com/apache/parquet-format/blob/74001e41f5c5a1856b29be115f9c992cab16a4bf/src/main/thrift/parquet.thrift#L1368-L1373 [closing-out-2.0]: https://lists.apache.org/thread/0bdyyb7qobrxx94x8v7t5z7g2ksnpyr2 +[encrypted footer]: https://github.com/apache/parquet-format/blob/master/Encryption.md#54-encrypted-footer-mode +[plaintext footer]: https://github.com/apache/parquet-format/blob/master/Encryption.md#55-plaintext-footer-mode ## Forward incompatible features by version @@ -75,9 +77,9 @@ Forward incompatible features and the format version each became available in: {{< format-versions table="forward_incompatible" >}} -> **Note:** Files with an [encrypted footer](https://github.com/apache/parquet-format/blob/master/Encryption.md#54-encrypted-footer-mode) use different magic bytes (`PARE` +> **Note:** Files with an [encrypted footer] use different magic bytes (`PARE` > instead of `PAR1`), making it clear to readers they must support modular -> encryption to read the file; [plaintext footer](https://github.com/apache/parquet-format/blob/master/Encryption.md#55-plaintext-footer-mode) files use `PAR1` so legacy +> encryption to read the file; [plaintext footer] files use `PAR1` so legacy > readers can still read their unencrypted columns. ## Forward compatible additions From 1fe107f18c5464373dbd7fa225e3c69d31112e60 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 24 Jun 2026 12:28:13 -0400 Subject: [PATCH 03/11] Restore format version explanatory note --- layouts/shortcodes/format-versions.html | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/layouts/shortcodes/format-versions.html b/layouts/shortcodes/format-versions.html index 1bebd453..c48a16de 100644 --- a/layouts/shortcodes/format-versions.html +++ b/layouts/shortcodes/format-versions.html @@ -73,6 +73,8 @@ {{- $sourceLabel = replace $sourceLabel "apache-parquet-format-" "" -}} {{- $sourceLabel = replace $sourceLabel "parquet-format-" "" -}} {{- end -}} + {{- $versionNote := $feature.version_note -}} + {{- $isApprovalNote := hasPrefix $versionNote "[Approved " -}} {{- if $featureUrl -}} @@ -80,6 +82,9 @@ {{- else -}} {{ $displayName | safeHTML }} {{- end -}} + {{- if and $versionNote (not $isApprovalNote) -}} +
{{ $versionNote | markdownify }} + {{- end -}} {{ $releasedIn }} @@ -88,11 +93,10 @@ {{ $sourceLabel }} - {{- $approvalNote := $feature.version_note -}} - {{- if hasPrefix $approvalNote "[Approved " -}} - {{- $approvalDate := replaceRE `^\[Approved ([0-9-]+)\]\(([^)]+)\)$` "$1" $approvalNote -}} - {{- $approvalUrl := replaceRE `^\[Approved ([0-9-]+)\]\(([^)]+)\)$` "$2" $approvalNote -}} - {{- if ne $approvalDate $approvalNote -}} + {{- if $isApprovalNote -}} + {{- $approvalDate := replaceRE `^\[Approved ([0-9-]+)\]\(([^)]+)\)$` "$1" $versionNote -}} + {{- $approvalUrl := replaceRE `^\[Approved ([0-9-]+)\]\(([^)]+)\)$` "$2" $versionNote -}} + {{- if ne $approvalDate $versionNote -}} {{ $approvalDate }} {{- end -}} {{- end -}} From 5875d4f7eb078026b2dbba0376dc840b8dae87ff Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 24 Jun 2026 12:30:44 -0400 Subject: [PATCH 04/11] Revert "Restore format version explanatory note" This reverts commit 1fe107f18c5464373dbd7fa225e3c69d31112e60. --- layouts/shortcodes/format-versions.html | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/layouts/shortcodes/format-versions.html b/layouts/shortcodes/format-versions.html index c48a16de..1bebd453 100644 --- a/layouts/shortcodes/format-versions.html +++ b/layouts/shortcodes/format-versions.html @@ -73,8 +73,6 @@ {{- $sourceLabel = replace $sourceLabel "apache-parquet-format-" "" -}} {{- $sourceLabel = replace $sourceLabel "parquet-format-" "" -}} {{- end -}} - {{- $versionNote := $feature.version_note -}} - {{- $isApprovalNote := hasPrefix $versionNote "[Approved " -}} {{- if $featureUrl -}} @@ -82,9 +80,6 @@ {{- else -}} {{ $displayName | safeHTML }} {{- end -}} - {{- if and $versionNote (not $isApprovalNote) -}} -
{{ $versionNote | markdownify }} - {{- end -}} {{ $releasedIn }} @@ -93,10 +88,11 @@ {{ $sourceLabel }} - {{- if $isApprovalNote -}} - {{- $approvalDate := replaceRE `^\[Approved ([0-9-]+)\]\(([^)]+)\)$` "$1" $versionNote -}} - {{- $approvalUrl := replaceRE `^\[Approved ([0-9-]+)\]\(([^)]+)\)$` "$2" $versionNote -}} - {{- if ne $approvalDate $versionNote -}} + {{- $approvalNote := $feature.version_note -}} + {{- if hasPrefix $approvalNote "[Approved " -}} + {{- $approvalDate := replaceRE `^\[Approved ([0-9-]+)\]\(([^)]+)\)$` "$1" $approvalNote -}} + {{- $approvalUrl := replaceRE `^\[Approved ([0-9-]+)\]\(([^)]+)\)$` "$2" $approvalNote -}} + {{- if ne $approvalDate $approvalNote -}} {{ $approvalDate }} {{- end -}} {{- end -}} From fbd740626ef144a71a884476e9b32427eb99cd12 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 26 Jun 2026 13:08:53 -0400 Subject: [PATCH 05/11] Add comments about what shortcode is doing --- data/implementations/features/encodings.yaml | 6 ++++-- .../features/format-features.yaml | 18 ++++++++++++------ .../features/logical-types.yaml | 15 ++++++++++----- layouts/shortcodes/format-versions.html | 14 ++++++++++++++ 4 files changed, 40 insertions(+), 13 deletions(-) diff --git a/data/implementations/features/encodings.yaml b/data/implementations/features/encodings.yaml index ac31a5db..205c3618 100644 --- a/data/implementations/features/encodings.yaml +++ b/data/implementations/features/encodings.yaml @@ -55,7 +55,8 @@ features: release_date: '2020-01-13' feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.7.0...apache-parquet-format-2.8.0 - version_note: "[Approved 2019-12-03](https://lists.apache.org/thread/xs5qt2odm299pxgqb22mty2csc1so5yr)" + approval_date: '2019-12-03' + approval_url: https://lists.apache.org/thread/xs5qt2odm299pxgqb22mty2csc1so5yr - id: encoding-byte-stream-split-extended display_name: BYTE_STREAM_SPLIT (Additional Types) spec_url: https://github.com/apache/parquet-format/commit/e517ac4dbe08d518eb5c2e58576d4c711973db94 @@ -65,4 +66,5 @@ features: release_date: '2024-03-21' feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 - version_note: "[Approved 2024-03-18](https://lists.apache.org/thread/nlsj0ftxy7y4ov1678rgy5zc7dmogg6q)" + approval_date: '2024-03-18' + approval_url: https://lists.apache.org/thread/nlsj0ftxy7y4ov1678rgy5zc7dmogg6q diff --git a/data/implementations/features/format-features.yaml b/data/implementations/features/format-features.yaml index b25e3438..999a6f7f 100644 --- a/data/implementations/features/format-features.yaml +++ b/data/implementations/features/format-features.yaml @@ -8,7 +8,8 @@ features: release_date: '2019-09-25' feature_url: https://github.com/apache/parquet-format/blob/master/BloomFilter.md source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.6.0...apache-parquet-format-2.7.0 - version_note: "[Approved 2019-09-09](https://lists.apache.org/thread/ktdx1xp0d2gjfgkcvd29zxvt3cgg88bo)" + approval_date: '2019-09-09' + approval_url: https://lists.apache.org/thread/ktdx1xp0d2gjfgkcvd29zxvt3cgg88bo - id: format-bloom-filter-length display_name: Bloom filter length note: 'In parquet.thrift: ColumnMetaData->bloom_filter_length' @@ -40,7 +41,8 @@ features: release_date: '2019-09-25' feature_url: https://github.com/apache/parquet-format/blob/master/Encryption.md source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.6.0...apache-parquet-format-2.7.0 - version_note: "[Approved 2019-01-16](https://lists.apache.org/thread/l8zcwnbrnhjh3w2k1lyb0v6ct5lnzr0h)" + approval_date: '2019-01-16' + approval_url: https://lists.apache.org/thread/l8zcwnbrnhjh3w2k1lyb0v6ct5lnzr0h - id: format-size-statistics display_name: Size statistics note: 'In parquet.thrift: ColumnMetaData->size_statistics' @@ -49,7 +51,8 @@ features: release_date: '2023-11-20' feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L202 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.9.0...apache-parquet-format-2.10.0 - version_note: "[Approved 2023-11-14](https://lists.apache.org/thread/wgobz41mfldbhqpg9q4mdwypghg2cxg2)" + approval_date: '2023-11-14' + approval_url: https://lists.apache.org/thread/wgobz41mfldbhqpg9q4mdwypghg2cxg2 - id: format-data-page-v2 display_name: Data Page V2 note: 'In parquet.thrift: DataPageHeaderV2' @@ -71,7 +74,8 @@ features: released_in: 2.11.0 feature_url: https://github.com/apache/parquet-format/blob/master/Geospatial.md#statistics source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 - version_note: "[Approved 2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" + approval_date: '2025-02-09' + approval_url: https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo - id: format-binary-protocol-extensions display_name: Binary protocol extensions implementation_status: false @@ -79,7 +83,8 @@ features: released_in: 2.11.0 feature_url: https://github.com/apache/parquet-format/blob/master/BinaryProtocolExtensions.md source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 - version_note: "[Approved 2024-09-06](https://lists.apache.org/thread/x3472kldrq5kjnld9ztj1jozz25f40hg)" + approval_date: '2024-09-06' + approval_url: https://lists.apache.org/thread/x3472kldrq5kjnld9ztj1jozz25f40hg - id: format-ieee-754-total-order-and-nan-counts display_name: IEEE 754 total order and NaN counts implementation_status: false @@ -87,4 +92,5 @@ features: released_in: 2.13.0 feature_url: https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift#L1061 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.12.0...apache-parquet-format-2.13.0 - version_note: "[Approved 2026-05-26](https://lists.apache.org/thread/h0k0hqo0sojqphnbnrkp8b0gmwdzq9on)" + approval_date: '2026-05-26' + approval_url: https://lists.apache.org/thread/h0k0hqo0sojqphnbnrkp8b0gmwdzq9on diff --git a/data/implementations/features/logical-types.yaml b/data/implementations/features/logical-types.yaml index fced412c..5bd79525 100644 --- a/data/implementations/features/logical-types.yaml +++ b/data/implementations/features/logical-types.yaml @@ -60,7 +60,8 @@ features: release_date: '2023-11-20' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#float16 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.9.0...apache-parquet-format-2.10.0 - version_note: "[Approved 2023-10-13](https://lists.apache.org/thread/gyvqcx9ssxkjlrwogqwy7n4z6ofdm871)" + approval_date: '2023-10-13' + approval_url: https://lists.apache.org/thread/gyvqcx9ssxkjlrwogqwy7n4z6ofdm871 - id: logical-date display_name: DATE version_display_name: DATE (INT32) @@ -115,7 +116,8 @@ features: release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.11.0...apache-parquet-format-2.12.0 - version_note: "[Approved 2025-08-24](https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7)" + approval_date: '2025-08-24' + approval_url: https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7 - id: logical-geometry display_name: GEOMETRY spec_url: https://github.com/apache/parquet-format/blob/master/Geospatial.md#logical-types @@ -125,7 +127,8 @@ features: release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geometry source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 - version_note: "[Approved 2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" + approval_date: '2025-02-09' + approval_url: https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo - id: logical-geography display_name: GEOGRAPHY spec_url: https://github.com/apache/parquet-format/blob/master/Geospatial.md#logical-types @@ -135,7 +138,8 @@ features: release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geography source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 - version_note: "[Approved 2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" + approval_date: '2025-02-09' + approval_url: https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo - id: logical-list display_name: LIST compatibility: forward_compatible @@ -176,4 +180,5 @@ features: released_in: 2.12.0 feature_url: https://github.com/apache/parquet-format/blob/master/VariantShredding.md source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.11.0...apache-parquet-format-2.12.0 - version_note: "[Approved 2025-08-24](https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7)" + approval_date: '2025-08-24' + approval_url: https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7 diff --git a/layouts/shortcodes/format-versions.html b/layouts/shortcodes/format-versions.html index 1bebd453..375d9e38 100644 --- a/layouts/shortcodes/format-versions.html +++ b/layouts/shortcodes/format-versions.html @@ -9,6 +9,13 @@ {{- errorf "unknown format versions table %q" $tableName -}} {{- end -}} +{{- /* + Collect features from each category that belong in the requested compatibility table. + Hugo sorts strings lexicographically, so build a zero-padded numeric sort key from + released_in before sorting. For example, "0002.0010.0000" sorts after + "0002.0009.0000", while plain string sorting would place "2.10.0" before "2.9.0". + Append the feature id to make ordering stable when multiple features share a release. +*/ -}} {{- $features := slice -}} {{- range site.Data.implementations.categories -}} {{- $featuresData := index site.Data.implementations.features .id -}} @@ -55,6 +62,13 @@ {{- $featureUrl = $feature.spec_url -}} {{- end -}} {{- $releasedIn := printf "%v" $feature.released_in -}} + {{- /* + Determine "$releaseTag" from the corresponding parquet-format GitHub release tag. + Older parquet-format releases used the tag prefix "parquet-format-", for example: + https://github.com/apache/parquet-format/releases/tag/parquet-format-2.1.0 + Newer releases use the tag prefix "apache-parquet-format-", for example: + https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.2.0 + */ -}} {{- $legacyReleaseTags := slice "1.0.0" "2.0.0" "2.1.0" -}} {{- $releaseTag := printf "apache-parquet-format-%s" $releasedIn -}} {{- if in $legacyReleaseTags $releasedIn -}} From e7dc3b0b8b329b9ac6db582c249beda8196abd3b Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 26 Jun 2026 13:12:46 -0400 Subject: [PATCH 06/11] Make approval / dates expicit and simplfy short code --- data/implementations/features/encodings.yaml | 6 ++---- .../features/format-features.yaml | 18 ++++++------------ .../features/logical-types.yaml | 15 +++++---------- layouts/shortcodes/format-versions.html | 9 ++------- 4 files changed, 15 insertions(+), 33 deletions(-) diff --git a/data/implementations/features/encodings.yaml b/data/implementations/features/encodings.yaml index 205c3618..a3138ee1 100644 --- a/data/implementations/features/encodings.yaml +++ b/data/implementations/features/encodings.yaml @@ -55,8 +55,7 @@ features: release_date: '2020-01-13' feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.7.0...apache-parquet-format-2.8.0 - approval_date: '2019-12-03' - approval_url: https://lists.apache.org/thread/xs5qt2odm299pxgqb22mty2csc1so5yr + approval: "[2019-12-03](https://lists.apache.org/thread/xs5qt2odm299pxgqb22mty2csc1so5yr)" - id: encoding-byte-stream-split-extended display_name: BYTE_STREAM_SPLIT (Additional Types) spec_url: https://github.com/apache/parquet-format/commit/e517ac4dbe08d518eb5c2e58576d4c711973db94 @@ -66,5 +65,4 @@ features: release_date: '2024-03-21' feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 - approval_date: '2024-03-18' - approval_url: https://lists.apache.org/thread/nlsj0ftxy7y4ov1678rgy5zc7dmogg6q + approval: "[2024-03-18](https://lists.apache.org/thread/nlsj0ftxy7y4ov1678rgy5zc7dmogg6q)" diff --git a/data/implementations/features/format-features.yaml b/data/implementations/features/format-features.yaml index 999a6f7f..31ff19de 100644 --- a/data/implementations/features/format-features.yaml +++ b/data/implementations/features/format-features.yaml @@ -8,8 +8,7 @@ features: release_date: '2019-09-25' feature_url: https://github.com/apache/parquet-format/blob/master/BloomFilter.md source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.6.0...apache-parquet-format-2.7.0 - approval_date: '2019-09-09' - approval_url: https://lists.apache.org/thread/ktdx1xp0d2gjfgkcvd29zxvt3cgg88bo + approval: "[2019-09-09](https://lists.apache.org/thread/ktdx1xp0d2gjfgkcvd29zxvt3cgg88bo)" - id: format-bloom-filter-length display_name: Bloom filter length note: 'In parquet.thrift: ColumnMetaData->bloom_filter_length' @@ -41,8 +40,7 @@ features: release_date: '2019-09-25' feature_url: https://github.com/apache/parquet-format/blob/master/Encryption.md source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.6.0...apache-parquet-format-2.7.0 - approval_date: '2019-01-16' - approval_url: https://lists.apache.org/thread/l8zcwnbrnhjh3w2k1lyb0v6ct5lnzr0h + approval: "[2019-01-16](https://lists.apache.org/thread/l8zcwnbrnhjh3w2k1lyb0v6ct5lnzr0h)" - id: format-size-statistics display_name: Size statistics note: 'In parquet.thrift: ColumnMetaData->size_statistics' @@ -51,8 +49,7 @@ features: release_date: '2023-11-20' feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L202 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.9.0...apache-parquet-format-2.10.0 - approval_date: '2023-11-14' - approval_url: https://lists.apache.org/thread/wgobz41mfldbhqpg9q4mdwypghg2cxg2 + approval: "[2023-11-14](https://lists.apache.org/thread/wgobz41mfldbhqpg9q4mdwypghg2cxg2)" - id: format-data-page-v2 display_name: Data Page V2 note: 'In parquet.thrift: DataPageHeaderV2' @@ -74,8 +71,7 @@ features: released_in: 2.11.0 feature_url: https://github.com/apache/parquet-format/blob/master/Geospatial.md#statistics source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 - approval_date: '2025-02-09' - approval_url: https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo + approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: format-binary-protocol-extensions display_name: Binary protocol extensions implementation_status: false @@ -83,8 +79,7 @@ features: released_in: 2.11.0 feature_url: https://github.com/apache/parquet-format/blob/master/BinaryProtocolExtensions.md source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 - approval_date: '2024-09-06' - approval_url: https://lists.apache.org/thread/x3472kldrq5kjnld9ztj1jozz25f40hg + approval: "[2024-09-06](https://lists.apache.org/thread/x3472kldrq5kjnld9ztj1jozz25f40hg)" - id: format-ieee-754-total-order-and-nan-counts display_name: IEEE 754 total order and NaN counts implementation_status: false @@ -92,5 +87,4 @@ features: released_in: 2.13.0 feature_url: https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift#L1061 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.12.0...apache-parquet-format-2.13.0 - approval_date: '2026-05-26' - approval_url: https://lists.apache.org/thread/h0k0hqo0sojqphnbnrkp8b0gmwdzq9on + approval: "[2026-05-26](https://lists.apache.org/thread/h0k0hqo0sojqphnbnrkp8b0gmwdzq9on)" diff --git a/data/implementations/features/logical-types.yaml b/data/implementations/features/logical-types.yaml index 5bd79525..b581d876 100644 --- a/data/implementations/features/logical-types.yaml +++ b/data/implementations/features/logical-types.yaml @@ -60,8 +60,7 @@ features: release_date: '2023-11-20' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#float16 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.9.0...apache-parquet-format-2.10.0 - approval_date: '2023-10-13' - approval_url: https://lists.apache.org/thread/gyvqcx9ssxkjlrwogqwy7n4z6ofdm871 + approval: "[2023-10-13](https://lists.apache.org/thread/gyvqcx9ssxkjlrwogqwy7n4z6ofdm871)" - id: logical-date display_name: DATE version_display_name: DATE (INT32) @@ -116,8 +115,7 @@ features: release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.11.0...apache-parquet-format-2.12.0 - approval_date: '2025-08-24' - approval_url: https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7 + approval: "[2025-08-24](https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7)" - id: logical-geometry display_name: GEOMETRY spec_url: https://github.com/apache/parquet-format/blob/master/Geospatial.md#logical-types @@ -127,8 +125,7 @@ features: release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geometry source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 - approval_date: '2025-02-09' - approval_url: https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo + approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: logical-geography display_name: GEOGRAPHY spec_url: https://github.com/apache/parquet-format/blob/master/Geospatial.md#logical-types @@ -138,8 +135,7 @@ features: release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geography source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 - approval_date: '2025-02-09' - approval_url: https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo + approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: logical-list display_name: LIST compatibility: forward_compatible @@ -180,5 +176,4 @@ features: released_in: 2.12.0 feature_url: https://github.com/apache/parquet-format/blob/master/VariantShredding.md source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.11.0...apache-parquet-format-2.12.0 - approval_date: '2025-08-24' - approval_url: https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7 + approval: "[2025-08-24](https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7)" diff --git a/layouts/shortcodes/format-versions.html b/layouts/shortcodes/format-versions.html index 375d9e38..bde5bf23 100644 --- a/layouts/shortcodes/format-versions.html +++ b/layouts/shortcodes/format-versions.html @@ -102,13 +102,8 @@ {{ $sourceLabel }} - {{- $approvalNote := $feature.version_note -}} - {{- if hasPrefix $approvalNote "[Approved " -}} - {{- $approvalDate := replaceRE `^\[Approved ([0-9-]+)\]\(([^)]+)\)$` "$1" $approvalNote -}} - {{- $approvalUrl := replaceRE `^\[Approved ([0-9-]+)\]\(([^)]+)\)$` "$2" $approvalNote -}} - {{- if ne $approvalDate $approvalNote -}} - {{ $approvalDate }} - {{- end -}} + {{- with $feature.approval -}} + {{ . | markdownify }} {{- end -}} From 7ef051a0361137e61d7338a408079a3644056c71 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 26 Jun 2026 13:16:27 -0400 Subject: [PATCH 07/11] simplify short code again --- .../features/compressions.yaml | 16 ++++++ data/implementations/features/encodings.yaml | 20 ++++++++ .../features/format-features.yaml | 22 ++++++++ .../features/logical-types.yaml | 50 +++++++++++++++++++ .../features/physical-types.yaml | 16 ++++++ layouts/shortcodes/format-versions.html | 38 ++------------ layouts/shortcodes/implementation-status.html | 8 +-- 7 files changed, 129 insertions(+), 41 deletions(-) diff --git a/data/implementations/features/compressions.yaml b/data/implementations/features/compressions.yaml index 2f523ac5..44e8bbce 100644 --- a/data/implementations/features/compressions.yaml +++ b/data/implementations/features/compressions.yaml @@ -4,47 +4,63 @@ features: display_name: UNCOMPRESSED compatibility: forward_incompatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#uncompressed source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 - id: compression-brotli display_name: BROTLI compatibility: forward_incompatible released_in: 2.4.0 + release_tag: apache-parquet-format-2.4.0 feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#brotli source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 + release_display: 2.3.1..2.4.0 - id: compression-gzip display_name: GZIP compatibility: forward_incompatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#gzip source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 - id: compression-lz4-deprecated display_name: LZ4 (deprecated) compatibility: forward_incompatible released_in: 2.4.0 + release_tag: apache-parquet-format-2.4.0 feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lz4 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 + release_display: 2.3.1..2.4.0 - id: compression-lz4-raw display_name: LZ4_RAW compatibility: forward_incompatible released_in: 2.9.0 + release_tag: apache-parquet-format-2.9.0 feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lz4_raw source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.8.0...apache-parquet-format-2.9.0 + release_display: 2.8.0..2.9.0 - id: compression-lzo display_name: LZO compatibility: forward_incompatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lzo source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 - id: compression-snappy display_name: SNAPPY compatibility: forward_incompatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#snappy source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 - id: compression-zstd display_name: ZSTD compatibility: forward_incompatible released_in: 2.4.0 + release_tag: apache-parquet-format-2.4.0 feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#zstd source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 + release_display: 2.3.1..2.4.0 diff --git a/data/implementations/features/encodings.yaml b/data/implementations/features/encodings.yaml index a3138ee1..d749d083 100644 --- a/data/implementations/features/encodings.yaml +++ b/data/implementations/features/encodings.yaml @@ -4,57 +4,75 @@ features: display_name: PLAIN compatibility: forward_incompatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#plain-plain--0 source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 - id: encoding-plain-dictionary display_name: PLAIN_DICTIONARY compatibility: forward_incompatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 - id: encoding-rle-dictionary display_name: RLE_DICTIONARY compatibility: forward_incompatible released_in: 2.0.0 + release_tag: parquet-format-2.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 + release_display: 1.0.0..2.0.0 - id: encoding-rle display_name: RLE compatibility: forward_incompatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3 source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 - id: encoding-bit-packed display_name: BIT_PACKED (deprecated) compatibility: forward_incompatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#bit-packed-deprecated-bit_packed--4 source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 - id: encoding-delta-binary-packed display_name: DELTA_BINARY_PACKED compatibility: forward_incompatible released_in: 2.0.0 + release_tag: parquet-format-2.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-encoding-delta_binary_packed--5 source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 + release_display: 1.0.0..2.0.0 - id: encoding-delta-length-byte-array display_name: DELTA_LENGTH_BYTE_ARRAY compatibility: forward_incompatible released_in: 2.0.0 + release_tag: parquet-format-2.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-length-byte-array-delta_length_byte_array--6 source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 + release_display: 1.0.0..2.0.0 - id: encoding-delta-byte-array display_name: DELTA_BYTE_ARRAY compatibility: forward_incompatible released_in: 2.0.0 + release_tag: parquet-format-2.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-strings-delta_byte_array--7 source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 + release_display: 1.0.0..2.0.0 - id: encoding-byte-stream-split display_name: BYTE_STREAM_SPLIT compatibility: forward_incompatible released_in: 2.8.0 + release_tag: apache-parquet-format-2.8.0 release_date: '2020-01-13' feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.7.0...apache-parquet-format-2.8.0 + release_display: 2.7.0..2.8.0 approval: "[2019-12-03](https://lists.apache.org/thread/xs5qt2odm299pxgqb22mty2csc1so5yr)" - id: encoding-byte-stream-split-extended display_name: BYTE_STREAM_SPLIT (Additional Types) @@ -62,7 +80,9 @@ features: version_display_name: BYTE_STREAM_SPLIT
(Additional Types) compatibility: forward_incompatible released_in: 2.11.0 + release_tag: apache-parquet-format-2.11.0 release_date: '2024-03-21' feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 + release_display: 2.10.0..2.11.0 approval: "[2024-03-18](https://lists.apache.org/thread/nlsj0ftxy7y4ov1678rgy5zc7dmogg6q)" diff --git a/data/implementations/features/format-features.yaml b/data/implementations/features/format-features.yaml index 31ff19de..27b76007 100644 --- a/data/implementations/features/format-features.yaml +++ b/data/implementations/features/format-features.yaml @@ -5,17 +5,21 @@ features: spec_url: https://github.com/apache/parquet-format/blob/master/BloomFilter.md compatibility: forward_compatible released_in: 2.7.0 + release_tag: apache-parquet-format-2.7.0 release_date: '2019-09-25' feature_url: https://github.com/apache/parquet-format/blob/master/BloomFilter.md source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.6.0...apache-parquet-format-2.7.0 + release_display: 2.6.0..2.7.0 approval: "[2019-09-09](https://lists.apache.org/thread/ktdx1xp0d2gjfgkcvd29zxvt3cgg88bo)" - id: format-bloom-filter-length display_name: Bloom filter length note: 'In parquet.thrift: ColumnMetaData->bloom_filter_length' compatibility: forward_compatible released_in: 2.10.0 + release_tag: apache-parquet-format-2.10.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L933 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.9.0...apache-parquet-format-2.10.0 + release_display: 2.9.0..2.10.0 - id: format-stats-min-max display_name: Statistics min_value, max_value - id: format-page-index @@ -23,68 +27,86 @@ features: spec_url: https://github.com/apache/parquet-format/blob/master/PageIndex.md compatibility: forward_compatible released_in: 2.4.0 + release_tag: apache-parquet-format-2.4.0 release_date: '2018-05-04' feature_url: https://github.com/apache/parquet-format/blob/master/PageIndex.md source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 + release_display: 2.3.1..2.4.0 - id: format-page-crc32 display_name: Page CRC32 checksum compatibility: forward_compatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L829 source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 - id: format-modular-encryption display_name: Modular encryption spec_url: https://github.com/apache/parquet-format/blob/master/Encryption.md compatibility: forward_incompatible released_in: 2.7.0 + release_tag: apache-parquet-format-2.7.0 release_date: '2019-09-25' feature_url: https://github.com/apache/parquet-format/blob/master/Encryption.md source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.6.0...apache-parquet-format-2.7.0 + release_display: 2.6.0..2.7.0 approval: "[2019-01-16](https://lists.apache.org/thread/l8zcwnbrnhjh3w2k1lyb0v6ct5lnzr0h)" - id: format-size-statistics display_name: Size statistics note: 'In parquet.thrift: ColumnMetaData->size_statistics' compatibility: forward_compatible released_in: 2.10.0 + release_tag: apache-parquet-format-2.10.0 release_date: '2023-11-20' feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L202 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.9.0...apache-parquet-format-2.10.0 + release_display: 2.9.0..2.10.0 approval: "[2023-11-14](https://lists.apache.org/thread/wgobz41mfldbhqpg9q4mdwypghg2cxg2)" - id: format-data-page-v2 display_name: Data Page V2 note: 'In parquet.thrift: DataPageHeaderV2' compatibility: forward_incompatible released_in: 2.0.0 + release_tag: parquet-format-2.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L724 source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 + release_display: 1.0.0..2.0.0 - id: format-data-page-v1 display_name: Data Page V1 implementation_status: false compatibility: forward_incompatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L671 source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 - id: format-geospatial-statistics display_name: Geospatial statistics implementation_status: false compatibility: forward_compatible released_in: 2.11.0 + release_tag: apache-parquet-format-2.11.0 feature_url: https://github.com/apache/parquet-format/blob/master/Geospatial.md#statistics source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 + release_display: 2.10.0..2.11.0 approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: format-binary-protocol-extensions display_name: Binary protocol extensions implementation_status: false compatibility: forward_compatible released_in: 2.11.0 + release_tag: apache-parquet-format-2.11.0 feature_url: https://github.com/apache/parquet-format/blob/master/BinaryProtocolExtensions.md source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 + release_display: 2.10.0..2.11.0 approval: "[2024-09-06](https://lists.apache.org/thread/x3472kldrq5kjnld9ztj1jozz25f40hg)" - id: format-ieee-754-total-order-and-nan-counts display_name: IEEE 754 total order and NaN counts implementation_status: false compatibility: forward_compatible released_in: 2.13.0 + release_tag: apache-parquet-format-2.13.0 feature_url: https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift#L1061 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.12.0...apache-parquet-format-2.13.0 + release_display: 2.12.0..2.13.0 approval: "[2026-05-26](https://lists.apache.org/thread/h0k0hqo0sojqphnbnrkp8b0gmwdzq9on)" diff --git a/data/implementations/features/logical-types.yaml b/data/implementations/features/logical-types.yaml index b581d876..d9c9aba4 100644 --- a/data/implementations/features/logical-types.yaml +++ b/data/implementations/features/logical-types.yaml @@ -5,116 +5,150 @@ features: version_display_name: STRING (BYTE_ARRAY) compatibility: forward_compatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#string source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 - id: logical-enum display_name: ENUM version_display_name: ENUM (BYTE_ARRAY) compatibility: forward_compatible released_in: 2.0.0 + release_tag: parquet-format-2.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#enum source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 + release_display: 1.0.0..2.0.0 - id: logical-uuid display_name: UUID version_display_name: UUID (FIXED_LEN_BYTE_ARRAY(16)) compatibility: forward_compatible released_in: 2.6.0 + release_tag: apache-parquet-format-2.6.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#uuid source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.5.0...apache-parquet-format-2.6.0 + release_display: 2.5.0..2.6.0 - id: logical-int-types display_name: 8, 16, 32, 64 bit signed and unsigned INT version_display_name: Signed and unsigned integer logical types (INT32, INT64) compatibility: forward_compatible released_in: 2.2.0 + release_tag: apache-parquet-format-2.2.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#signed-integers source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 + release_display: 2.1.0..2.2.0 - id: logical-decimal-int32 display_name: DECIMAL (INT32) compatibility: forward_compatible released_in: 2.1.0 + release_tag: parquet-format-2.1.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.0.0...parquet-format-2.1.0 + release_display: 2.0.0..2.1.0 - id: logical-decimal-int64 display_name: DECIMAL (INT64) compatibility: forward_compatible released_in: 2.1.0 + release_tag: parquet-format-2.1.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.0.0...parquet-format-2.1.0 + release_display: 2.0.0..2.1.0 - id: logical-decimal-byte-array display_name: DECIMAL (BYTE_ARRAY) compatibility: forward_compatible released_in: 2.1.0 + release_tag: parquet-format-2.1.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.0.0...parquet-format-2.1.0 + release_display: 2.0.0..2.1.0 - id: logical-decimal-fixed-len-byte-array display_name: DECIMAL (FIXED_LEN_BYTE_ARRAY) compatibility: forward_compatible released_in: 2.1.0 + release_tag: parquet-format-2.1.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.0.0...parquet-format-2.1.0 + release_display: 2.0.0..2.1.0 - id: logical-float16 display_name: FLOAT16 version_display_name: FLOAT16 (FIXED_LEN_BYTE_ARRAY(2)) compatibility: forward_compatible released_in: 2.10.0 + release_tag: apache-parquet-format-2.10.0 release_date: '2023-11-20' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#float16 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.9.0...apache-parquet-format-2.10.0 + release_display: 2.9.0..2.10.0 approval: "[2023-10-13](https://lists.apache.org/thread/gyvqcx9ssxkjlrwogqwy7n4z6ofdm871)" - id: logical-date display_name: DATE version_display_name: DATE (INT32) compatibility: forward_compatible released_in: 2.2.0 + release_tag: apache-parquet-format-2.2.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#date source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 + release_display: 2.1.0..2.2.0 - id: logical-time-int32 display_name: TIME (INT32) compatibility: forward_compatible released_in: 2.2.0 + release_tag: apache-parquet-format-2.2.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 + release_display: 2.1.0..2.2.0 - id: logical-time-int64 display_name: TIME (INT64) compatibility: forward_compatible released_in: 2.4.0 + release_tag: apache-parquet-format-2.4.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 + release_display: 2.3.1..2.4.0 - id: logical-timestamp-int64 display_name: TIMESTAMP (INT64) compatibility: forward_compatible released_in: 2.2.0 + release_tag: apache-parquet-format-2.2.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#timestamp source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 + release_display: 2.1.0..2.2.0 - id: logical-interval display_name: INTERVAL version_display_name: INTERVAL (FIXED_LEN_BYTE_ARRAY(12)) compatibility: forward_compatible released_in: 2.2.0 + release_tag: apache-parquet-format-2.2.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#interval source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 + release_display: 2.1.0..2.2.0 - id: logical-json display_name: JSON version_display_name: JSON (BYTE_ARRAY) compatibility: forward_compatible released_in: 2.2.0 + release_tag: apache-parquet-format-2.2.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#json source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 + release_display: 2.1.0..2.2.0 - id: logical-bson display_name: BSON version_display_name: BSON (BYTE_ARRAY) compatibility: forward_compatible released_in: 2.2.0 + release_tag: apache-parquet-format-2.2.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#bson source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 + release_display: 2.1.0..2.2.0 - id: logical-variant display_name: VARIANT spec_url: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md compatibility: forward_compatible released_in: 2.12.0 + release_tag: apache-parquet-format-2.12.0 release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.11.0...apache-parquet-format-2.12.0 + release_display: 2.11.0..2.12.0 approval: "[2025-08-24](https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7)" - id: logical-geometry display_name: GEOMETRY @@ -122,9 +156,11 @@ features: version_display_name: GEOMETRY (BYTE_ARRAY) compatibility: forward_compatible released_in: 2.11.0 + release_tag: apache-parquet-format-2.11.0 release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geometry source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 + release_display: 2.10.0..2.11.0 approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: logical-geography display_name: GEOGRAPHY @@ -132,48 +168,62 @@ features: version_display_name: GEOGRAPHY (BYTE_ARRAY) compatibility: forward_compatible released_in: 2.11.0 + release_tag: apache-parquet-format-2.11.0 release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geography source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 + release_display: 2.10.0..2.11.0 approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: logical-list display_name: LIST compatibility: forward_compatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 - id: logical-map display_name: MAP compatibility: forward_compatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#maps source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 - id: logical-unknown display_name: UNKNOWN (always null) compatibility: forward_compatible released_in: 2.4.0 + release_tag: apache-parquet-format-2.4.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#unknown-always-null source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 + release_display: 2.3.1..2.4.0 - id: logical-type-union display_name: LogicalType union implementation_status: false compatibility: forward_compatible released_in: 2.4.0 + release_tag: apache-parquet-format-2.4.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L471 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 + release_display: 2.3.1..2.4.0 version_note: Supersedes `ConvertedType` enum
deprecated in [2.9.0](https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.9.0) - id: logical-nanosecond-time-timestamp display_name: Nanosecond TIME/TIMESTAMP implementation_status: false compatibility: forward_compatible released_in: 2.6.0 + release_tag: apache-parquet-format-2.6.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L352 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.5.0...apache-parquet-format-2.6.0 + release_display: 2.5.0..2.6.0 - id: logical-variant-shredding display_name: Variant shredding implementation_status: false compatibility: forward_compatible released_in: 2.12.0 + release_tag: apache-parquet-format-2.12.0 feature_url: https://github.com/apache/parquet-format/blob/master/VariantShredding.md source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.11.0...apache-parquet-format-2.12.0 + release_display: 2.11.0..2.12.0 approval: "[2025-08-24](https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7)" diff --git a/data/implementations/features/physical-types.yaml b/data/implementations/features/physical-types.yaml index c445fe13..38964758 100644 --- a/data/implementations/features/physical-types.yaml +++ b/data/implementations/features/physical-types.yaml @@ -4,49 +4,65 @@ features: display_name: BOOLEAN compatibility: forward_incompatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L33 source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 - id: physical-int32 display_name: INT32 compatibility: forward_incompatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L34 source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 - id: physical-int64 display_name: INT64 compatibility: forward_incompatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L35 source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 - id: physical-int96 display_name: INT96 note: This type is deprecated, but as of 2024 it's common in currently produced parquet files version_display_name: INT96 (deprecated) compatibility: forward_incompatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L36 source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 - id: physical-float display_name: FLOAT compatibility: forward_incompatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L37 source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 - id: physical-double display_name: DOUBLE compatibility: forward_incompatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L38 source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 - id: physical-byte-array display_name: BYTE_ARRAY compatibility: forward_incompatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L39 source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 - id: physical-fixed-len-byte-array display_name: FIXED_LEN_BYTE_ARRAY compatibility: forward_incompatible released_in: 1.0.0 + release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L40 source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + release_display: 1.0.0 diff --git a/layouts/shortcodes/format-versions.html b/layouts/shortcodes/format-versions.html index bde5bf23..7d535e81 100644 --- a/layouts/shortcodes/format-versions.html +++ b/layouts/shortcodes/format-versions.html @@ -53,40 +53,10 @@ {{- range $features }} {{- $feature := .feature -}} - {{- $displayName := $feature.display_name -}} - {{- if $feature.version_display_name -}} - {{- $displayName = $feature.version_display_name -}} - {{- end -}} - {{- $featureUrl := $feature.feature_url -}} - {{- if and (not $featureUrl) $feature.spec_url -}} - {{- $featureUrl = $feature.spec_url -}} - {{- end -}} + {{- $displayName := or $feature.version_display_name $feature.display_name -}} + {{- $featureUrl := or $feature.feature_url $feature.spec_url -}} {{- $releasedIn := printf "%v" $feature.released_in -}} - {{- /* - Determine "$releaseTag" from the corresponding parquet-format GitHub release tag. - Older parquet-format releases used the tag prefix "parquet-format-", for example: - https://github.com/apache/parquet-format/releases/tag/parquet-format-2.1.0 - Newer releases use the tag prefix "apache-parquet-format-", for example: - https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.2.0 - */ -}} - {{- $legacyReleaseTags := slice "1.0.0" "2.0.0" "2.1.0" -}} - {{- $releaseTag := printf "apache-parquet-format-%s" $releasedIn -}} - {{- if in $legacyReleaseTags $releasedIn -}} - {{- $releaseTag = printf "parquet-format-%s" $releasedIn -}} - {{- end -}} - {{- $releaseUrl := printf "https://github.com/apache/parquet-format/releases/tag/%s" $releaseTag -}} - {{- $sourceUrl := $feature.source_url -}} - {{- $sourceLabel := "" -}} - {{- if in $sourceUrl "/compare/" -}} - {{- $compareRange := replaceRE "^.*/compare/" "" $sourceUrl -}} - {{- $compareRange = replace $compareRange "apache-parquet-format-" "" -}} - {{- $compareRange = replace $compareRange "parquet-format-" "" -}} - {{- $sourceLabel = replace $compareRange "..." ".." -}} - {{- else -}} - {{- $sourceLabel = path.Base $sourceUrl -}} - {{- $sourceLabel = replace $sourceLabel "apache-parquet-format-" "" -}} - {{- $sourceLabel = replace $sourceLabel "parquet-format-" "" -}} - {{- end -}} + {{- $releaseUrl := printf "https://github.com/apache/parquet-format/releases/tag/%s" $feature.release_tag -}} {{- if $featureUrl -}} @@ -99,7 +69,7 @@ {{ $releasedIn }} - {{ $sourceLabel }} + {{ $feature.release_display }} {{- with $feature.approval -}} diff --git a/layouts/shortcodes/implementation-status.html b/layouts/shortcodes/implementation-status.html index 4dcc3ca6..d8c53050 100644 --- a/layouts/shortcodes/implementation-status.html +++ b/layouts/shortcodes/implementation-status.html @@ -78,13 +78,7 @@

{{ $category.name }} {{- else -}} {{- $year = substr $dateStr 0 4 -}} {{- end -}} - {{- $releasedIn := printf "%v" .released_in -}} - {{- $legacyReleaseTags := slice "1.0.0" "2.0.0" "2.1.0" -}} - {{- $releaseTag := printf "apache-parquet-format-%s" $releasedIn -}} - {{- if in $legacyReleaseTags $releasedIn -}} - {{- $releaseTag = printf "parquet-format-%s" $releasedIn -}} - {{- end -}} - {{- $releaseUrl := printf "https://github.com/apache/parquet-format/releases/tag/%s" $releaseTag -}} + {{- $releaseUrl := printf "https://github.com/apache/parquet-format/releases/tag/%s" .release_tag -}} ({{ $year }}) {{- end -}} From 75652eb840b8f6e55747c26bca87817a6f1daaaa Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 26 Jun 2026 13:24:52 -0400 Subject: [PATCH 08/11] simplify comments --- .../features/compressions.yaml | 24 ------ data/implementations/features/encodings.yaml | 30 -------- .../features/format-features.yaml | 33 -------- .../features/logical-types.yaml | 75 ------------------- .../features/physical-types.yaml | 24 ------ .../parquet-format-releases.yaml | 67 +++++++++++++++++ layouts/shortcodes/format-versions.html | 11 ++- layouts/shortcodes/implementation-status.html | 10 ++- 8 files changed, 85 insertions(+), 189 deletions(-) create mode 100644 data/implementations/parquet-format-releases.yaml diff --git a/data/implementations/features/compressions.yaml b/data/implementations/features/compressions.yaml index 44e8bbce..26a568cb 100644 --- a/data/implementations/features/compressions.yaml +++ b/data/implementations/features/compressions.yaml @@ -4,63 +4,39 @@ features: display_name: UNCOMPRESSED compatibility: forward_incompatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#uncompressed - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 - id: compression-brotli display_name: BROTLI compatibility: forward_incompatible released_in: 2.4.0 - release_tag: apache-parquet-format-2.4.0 feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#brotli - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 - release_display: 2.3.1..2.4.0 - id: compression-gzip display_name: GZIP compatibility: forward_incompatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#gzip - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 - id: compression-lz4-deprecated display_name: LZ4 (deprecated) compatibility: forward_incompatible released_in: 2.4.0 - release_tag: apache-parquet-format-2.4.0 feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lz4 - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 - release_display: 2.3.1..2.4.0 - id: compression-lz4-raw display_name: LZ4_RAW compatibility: forward_incompatible released_in: 2.9.0 - release_tag: apache-parquet-format-2.9.0 feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lz4_raw - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.8.0...apache-parquet-format-2.9.0 - release_display: 2.8.0..2.9.0 - id: compression-lzo display_name: LZO compatibility: forward_incompatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lzo - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 - id: compression-snappy display_name: SNAPPY compatibility: forward_incompatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#snappy - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 - id: compression-zstd display_name: ZSTD compatibility: forward_incompatible released_in: 2.4.0 - release_tag: apache-parquet-format-2.4.0 feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#zstd - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 - release_display: 2.3.1..2.4.0 diff --git a/data/implementations/features/encodings.yaml b/data/implementations/features/encodings.yaml index d749d083..9c478095 100644 --- a/data/implementations/features/encodings.yaml +++ b/data/implementations/features/encodings.yaml @@ -4,75 +4,48 @@ features: display_name: PLAIN compatibility: forward_incompatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#plain-plain--0 - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 - id: encoding-plain-dictionary display_name: PLAIN_DICTIONARY compatibility: forward_incompatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 - id: encoding-rle-dictionary display_name: RLE_DICTIONARY compatibility: forward_incompatible released_in: 2.0.0 - release_tag: parquet-format-2.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 - source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 - release_display: 1.0.0..2.0.0 - id: encoding-rle display_name: RLE compatibility: forward_incompatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3 - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 - id: encoding-bit-packed display_name: BIT_PACKED (deprecated) compatibility: forward_incompatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#bit-packed-deprecated-bit_packed--4 - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 - id: encoding-delta-binary-packed display_name: DELTA_BINARY_PACKED compatibility: forward_incompatible released_in: 2.0.0 - release_tag: parquet-format-2.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-encoding-delta_binary_packed--5 - source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 - release_display: 1.0.0..2.0.0 - id: encoding-delta-length-byte-array display_name: DELTA_LENGTH_BYTE_ARRAY compatibility: forward_incompatible released_in: 2.0.0 - release_tag: parquet-format-2.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-length-byte-array-delta_length_byte_array--6 - source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 - release_display: 1.0.0..2.0.0 - id: encoding-delta-byte-array display_name: DELTA_BYTE_ARRAY compatibility: forward_incompatible released_in: 2.0.0 - release_tag: parquet-format-2.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-strings-delta_byte_array--7 - source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 - release_display: 1.0.0..2.0.0 - id: encoding-byte-stream-split display_name: BYTE_STREAM_SPLIT compatibility: forward_incompatible released_in: 2.8.0 - release_tag: apache-parquet-format-2.8.0 release_date: '2020-01-13' feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.7.0...apache-parquet-format-2.8.0 - release_display: 2.7.0..2.8.0 approval: "[2019-12-03](https://lists.apache.org/thread/xs5qt2odm299pxgqb22mty2csc1so5yr)" - id: encoding-byte-stream-split-extended display_name: BYTE_STREAM_SPLIT (Additional Types) @@ -80,9 +53,6 @@ features: version_display_name: BYTE_STREAM_SPLIT
(Additional Types) compatibility: forward_incompatible released_in: 2.11.0 - release_tag: apache-parquet-format-2.11.0 release_date: '2024-03-21' feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 - release_display: 2.10.0..2.11.0 approval: "[2024-03-18](https://lists.apache.org/thread/nlsj0ftxy7y4ov1678rgy5zc7dmogg6q)" diff --git a/data/implementations/features/format-features.yaml b/data/implementations/features/format-features.yaml index 27b76007..0008d1b1 100644 --- a/data/implementations/features/format-features.yaml +++ b/data/implementations/features/format-features.yaml @@ -5,21 +5,15 @@ features: spec_url: https://github.com/apache/parquet-format/blob/master/BloomFilter.md compatibility: forward_compatible released_in: 2.7.0 - release_tag: apache-parquet-format-2.7.0 release_date: '2019-09-25' feature_url: https://github.com/apache/parquet-format/blob/master/BloomFilter.md - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.6.0...apache-parquet-format-2.7.0 - release_display: 2.6.0..2.7.0 approval: "[2019-09-09](https://lists.apache.org/thread/ktdx1xp0d2gjfgkcvd29zxvt3cgg88bo)" - id: format-bloom-filter-length display_name: Bloom filter length note: 'In parquet.thrift: ColumnMetaData->bloom_filter_length' compatibility: forward_compatible released_in: 2.10.0 - release_tag: apache-parquet-format-2.10.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L933 - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.9.0...apache-parquet-format-2.10.0 - release_display: 2.9.0..2.10.0 - id: format-stats-min-max display_name: Statistics min_value, max_value - id: format-page-index @@ -27,86 +21,59 @@ features: spec_url: https://github.com/apache/parquet-format/blob/master/PageIndex.md compatibility: forward_compatible released_in: 2.4.0 - release_tag: apache-parquet-format-2.4.0 release_date: '2018-05-04' feature_url: https://github.com/apache/parquet-format/blob/master/PageIndex.md - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 - release_display: 2.3.1..2.4.0 - id: format-page-crc32 display_name: Page CRC32 checksum compatibility: forward_compatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L829 - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 - id: format-modular-encryption display_name: Modular encryption spec_url: https://github.com/apache/parquet-format/blob/master/Encryption.md compatibility: forward_incompatible released_in: 2.7.0 - release_tag: apache-parquet-format-2.7.0 release_date: '2019-09-25' feature_url: https://github.com/apache/parquet-format/blob/master/Encryption.md - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.6.0...apache-parquet-format-2.7.0 - release_display: 2.6.0..2.7.0 approval: "[2019-01-16](https://lists.apache.org/thread/l8zcwnbrnhjh3w2k1lyb0v6ct5lnzr0h)" - id: format-size-statistics display_name: Size statistics note: 'In parquet.thrift: ColumnMetaData->size_statistics' compatibility: forward_compatible released_in: 2.10.0 - release_tag: apache-parquet-format-2.10.0 release_date: '2023-11-20' feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L202 - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.9.0...apache-parquet-format-2.10.0 - release_display: 2.9.0..2.10.0 approval: "[2023-11-14](https://lists.apache.org/thread/wgobz41mfldbhqpg9q4mdwypghg2cxg2)" - id: format-data-page-v2 display_name: Data Page V2 note: 'In parquet.thrift: DataPageHeaderV2' compatibility: forward_incompatible released_in: 2.0.0 - release_tag: parquet-format-2.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L724 - source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 - release_display: 1.0.0..2.0.0 - id: format-data-page-v1 display_name: Data Page V1 implementation_status: false compatibility: forward_incompatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L671 - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 - id: format-geospatial-statistics display_name: Geospatial statistics implementation_status: false compatibility: forward_compatible released_in: 2.11.0 - release_tag: apache-parquet-format-2.11.0 feature_url: https://github.com/apache/parquet-format/blob/master/Geospatial.md#statistics - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 - release_display: 2.10.0..2.11.0 approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: format-binary-protocol-extensions display_name: Binary protocol extensions implementation_status: false compatibility: forward_compatible released_in: 2.11.0 - release_tag: apache-parquet-format-2.11.0 feature_url: https://github.com/apache/parquet-format/blob/master/BinaryProtocolExtensions.md - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 - release_display: 2.10.0..2.11.0 approval: "[2024-09-06](https://lists.apache.org/thread/x3472kldrq5kjnld9ztj1jozz25f40hg)" - id: format-ieee-754-total-order-and-nan-counts display_name: IEEE 754 total order and NaN counts implementation_status: false compatibility: forward_compatible released_in: 2.13.0 - release_tag: apache-parquet-format-2.13.0 feature_url: https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift#L1061 - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.12.0...apache-parquet-format-2.13.0 - release_display: 2.12.0..2.13.0 approval: "[2026-05-26](https://lists.apache.org/thread/h0k0hqo0sojqphnbnrkp8b0gmwdzq9on)" diff --git a/data/implementations/features/logical-types.yaml b/data/implementations/features/logical-types.yaml index d9c9aba4..3cfe5490 100644 --- a/data/implementations/features/logical-types.yaml +++ b/data/implementations/features/logical-types.yaml @@ -5,150 +5,99 @@ features: version_display_name: STRING (BYTE_ARRAY) compatibility: forward_compatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#string - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 - id: logical-enum display_name: ENUM version_display_name: ENUM (BYTE_ARRAY) compatibility: forward_compatible released_in: 2.0.0 - release_tag: parquet-format-2.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#enum - source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 - release_display: 1.0.0..2.0.0 - id: logical-uuid display_name: UUID version_display_name: UUID (FIXED_LEN_BYTE_ARRAY(16)) compatibility: forward_compatible released_in: 2.6.0 - release_tag: apache-parquet-format-2.6.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#uuid - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.5.0...apache-parquet-format-2.6.0 - release_display: 2.5.0..2.6.0 - id: logical-int-types display_name: 8, 16, 32, 64 bit signed and unsigned INT version_display_name: Signed and unsigned integer logical types (INT32, INT64) compatibility: forward_compatible released_in: 2.2.0 - release_tag: apache-parquet-format-2.2.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#signed-integers - source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 - release_display: 2.1.0..2.2.0 - id: logical-decimal-int32 display_name: DECIMAL (INT32) compatibility: forward_compatible released_in: 2.1.0 - release_tag: parquet-format-2.1.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal - source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.0.0...parquet-format-2.1.0 - release_display: 2.0.0..2.1.0 - id: logical-decimal-int64 display_name: DECIMAL (INT64) compatibility: forward_compatible released_in: 2.1.0 - release_tag: parquet-format-2.1.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal - source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.0.0...parquet-format-2.1.0 - release_display: 2.0.0..2.1.0 - id: logical-decimal-byte-array display_name: DECIMAL (BYTE_ARRAY) compatibility: forward_compatible released_in: 2.1.0 - release_tag: parquet-format-2.1.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal - source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.0.0...parquet-format-2.1.0 - release_display: 2.0.0..2.1.0 - id: logical-decimal-fixed-len-byte-array display_name: DECIMAL (FIXED_LEN_BYTE_ARRAY) compatibility: forward_compatible released_in: 2.1.0 - release_tag: parquet-format-2.1.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal - source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.0.0...parquet-format-2.1.0 - release_display: 2.0.0..2.1.0 - id: logical-float16 display_name: FLOAT16 version_display_name: FLOAT16 (FIXED_LEN_BYTE_ARRAY(2)) compatibility: forward_compatible released_in: 2.10.0 - release_tag: apache-parquet-format-2.10.0 release_date: '2023-11-20' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#float16 - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.9.0...apache-parquet-format-2.10.0 - release_display: 2.9.0..2.10.0 approval: "[2023-10-13](https://lists.apache.org/thread/gyvqcx9ssxkjlrwogqwy7n4z6ofdm871)" - id: logical-date display_name: DATE version_display_name: DATE (INT32) compatibility: forward_compatible released_in: 2.2.0 - release_tag: apache-parquet-format-2.2.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#date - source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 - release_display: 2.1.0..2.2.0 - id: logical-time-int32 display_name: TIME (INT32) compatibility: forward_compatible released_in: 2.2.0 - release_tag: apache-parquet-format-2.2.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time - source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 - release_display: 2.1.0..2.2.0 - id: logical-time-int64 display_name: TIME (INT64) compatibility: forward_compatible released_in: 2.4.0 - release_tag: apache-parquet-format-2.4.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 - release_display: 2.3.1..2.4.0 - id: logical-timestamp-int64 display_name: TIMESTAMP (INT64) compatibility: forward_compatible released_in: 2.2.0 - release_tag: apache-parquet-format-2.2.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#timestamp - source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 - release_display: 2.1.0..2.2.0 - id: logical-interval display_name: INTERVAL version_display_name: INTERVAL (FIXED_LEN_BYTE_ARRAY(12)) compatibility: forward_compatible released_in: 2.2.0 - release_tag: apache-parquet-format-2.2.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#interval - source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 - release_display: 2.1.0..2.2.0 - id: logical-json display_name: JSON version_display_name: JSON (BYTE_ARRAY) compatibility: forward_compatible released_in: 2.2.0 - release_tag: apache-parquet-format-2.2.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#json - source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 - release_display: 2.1.0..2.2.0 - id: logical-bson display_name: BSON version_display_name: BSON (BYTE_ARRAY) compatibility: forward_compatible released_in: 2.2.0 - release_tag: apache-parquet-format-2.2.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#bson - source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 - release_display: 2.1.0..2.2.0 - id: logical-variant display_name: VARIANT spec_url: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md compatibility: forward_compatible released_in: 2.12.0 - release_tag: apache-parquet-format-2.12.0 release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.11.0...apache-parquet-format-2.12.0 - release_display: 2.11.0..2.12.0 approval: "[2025-08-24](https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7)" - id: logical-geometry display_name: GEOMETRY @@ -156,11 +105,8 @@ features: version_display_name: GEOMETRY (BYTE_ARRAY) compatibility: forward_compatible released_in: 2.11.0 - release_tag: apache-parquet-format-2.11.0 release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geometry - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 - release_display: 2.10.0..2.11.0 approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: logical-geography display_name: GEOGRAPHY @@ -168,62 +114,41 @@ features: version_display_name: GEOGRAPHY (BYTE_ARRAY) compatibility: forward_compatible released_in: 2.11.0 - release_tag: apache-parquet-format-2.11.0 release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geography - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 - release_display: 2.10.0..2.11.0 approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: logical-list display_name: LIST compatibility: forward_compatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 - id: logical-map display_name: MAP compatibility: forward_compatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#maps - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 - id: logical-unknown display_name: UNKNOWN (always null) compatibility: forward_compatible released_in: 2.4.0 - release_tag: apache-parquet-format-2.4.0 feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#unknown-always-null - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 - release_display: 2.3.1..2.4.0 - id: logical-type-union display_name: LogicalType union implementation_status: false compatibility: forward_compatible released_in: 2.4.0 - release_tag: apache-parquet-format-2.4.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L471 - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 - release_display: 2.3.1..2.4.0 version_note: Supersedes `ConvertedType` enum
deprecated in [2.9.0](https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.9.0) - id: logical-nanosecond-time-timestamp display_name: Nanosecond TIME/TIMESTAMP implementation_status: false compatibility: forward_compatible released_in: 2.6.0 - release_tag: apache-parquet-format-2.6.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L352 - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.5.0...apache-parquet-format-2.6.0 - release_display: 2.5.0..2.6.0 - id: logical-variant-shredding display_name: Variant shredding implementation_status: false compatibility: forward_compatible released_in: 2.12.0 - release_tag: apache-parquet-format-2.12.0 feature_url: https://github.com/apache/parquet-format/blob/master/VariantShredding.md - source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.11.0...apache-parquet-format-2.12.0 - release_display: 2.11.0..2.12.0 approval: "[2025-08-24](https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7)" diff --git a/data/implementations/features/physical-types.yaml b/data/implementations/features/physical-types.yaml index 38964758..2f0dddfd 100644 --- a/data/implementations/features/physical-types.yaml +++ b/data/implementations/features/physical-types.yaml @@ -4,65 +4,41 @@ features: display_name: BOOLEAN compatibility: forward_incompatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L33 - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 - id: physical-int32 display_name: INT32 compatibility: forward_incompatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L34 - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 - id: physical-int64 display_name: INT64 compatibility: forward_incompatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L35 - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 - id: physical-int96 display_name: INT96 note: This type is deprecated, but as of 2024 it's common in currently produced parquet files version_display_name: INT96 (deprecated) compatibility: forward_incompatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L36 - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 - id: physical-float display_name: FLOAT compatibility: forward_incompatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L37 - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 - id: physical-double display_name: DOUBLE compatibility: forward_incompatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L38 - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 - id: physical-byte-array display_name: BYTE_ARRAY compatibility: forward_incompatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L39 - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 - id: physical-fixed-len-byte-array display_name: FIXED_LEN_BYTE_ARRAY compatibility: forward_incompatible released_in: 1.0.0 - release_tag: parquet-format-1.0.0 feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L40 - source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - release_display: 1.0.0 diff --git a/data/implementations/parquet-format-releases.yaml b/data/implementations/parquet-format-releases.yaml new file mode 100644 index 00000000..30a62810 --- /dev/null +++ b/data/implementations/parquet-format-releases.yaml @@ -0,0 +1,67 @@ +# Metadata about releases of the apache/parquet-format repository. +# +# Each top-level key is a parquet-format version. The values describe the +# GitHub release tag and the source tree or compare link for that release. +# +# This is intentionally separate from data/implementations/features/*.yaml: +# many feature rows were released in the same parquet-format version, so they +# can reference the shared metadata with only their `released_in` value. +# +# The shortcodes use this file to build "Released in" and "Source" links in the +# implementation and format-version tables. +# +# Older releases used the `parquet-format-*` tag prefix. Newer releases use +# `apache-parquet-format-*`, so this map keeps that historical detail out of +# each individual feature row. +1.0.0: + tag: parquet-format-1.0.0 + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + source_display: 1.0.0 +2.0.0: + tag: parquet-format-2.0.0 + source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 + source_display: 1.0.0..2.0.0 +2.1.0: + tag: parquet-format-2.1.0 + source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.0.0...parquet-format-2.1.0 + source_display: 2.0.0..2.1.0 +2.2.0: + tag: apache-parquet-format-2.2.0 + source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 + source_display: 2.1.0..2.2.0 +2.4.0: + tag: apache-parquet-format-2.4.0 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 + source_display: 2.3.1..2.4.0 +2.6.0: + tag: apache-parquet-format-2.6.0 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.5.0...apache-parquet-format-2.6.0 + source_display: 2.5.0..2.6.0 +2.7.0: + tag: apache-parquet-format-2.7.0 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.6.0...apache-parquet-format-2.7.0 + source_display: 2.6.0..2.7.0 +2.8.0: + tag: apache-parquet-format-2.8.0 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.7.0...apache-parquet-format-2.8.0 + source_display: 2.7.0..2.8.0 +2.9.0: + tag: apache-parquet-format-2.9.0 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.8.0...apache-parquet-format-2.9.0 + source_display: 2.8.0..2.9.0 +2.10.0: + tag: apache-parquet-format-2.10.0 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.9.0...apache-parquet-format-2.10.0 + source_display: 2.9.0..2.10.0 +2.11.0: + tag: apache-parquet-format-2.11.0 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 + source_display: 2.10.0..2.11.0 +2.12.0: + tag: apache-parquet-format-2.12.0 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.11.0...apache-parquet-format-2.12.0 + source_display: 2.11.0..2.12.0 +2.13.0: + tag: apache-parquet-format-2.13.0 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.12.0...apache-parquet-format-2.13.0 + source_display: 2.12.0..2.13.0 diff --git a/layouts/shortcodes/format-versions.html b/layouts/shortcodes/format-versions.html index 7d535e81..5279d5ad 100644 --- a/layouts/shortcodes/format-versions.html +++ b/layouts/shortcodes/format-versions.html @@ -9,6 +9,8 @@ {{- errorf "unknown format versions table %q" $tableName -}} {{- end -}} +{{- /* Shared release-tag metadata keyed by feature released_in version. */ -}} +{{- $formatReleases := index site.Data.implementations "parquet-format-releases" -}} {{- /* Collect features from each category that belong in the requested compatibility table. Hugo sorts strings lexicographically, so build a zero-padded numeric sort key from @@ -56,7 +58,12 @@ {{- $displayName := or $feature.version_display_name $feature.display_name -}} {{- $featureUrl := or $feature.feature_url $feature.spec_url -}} {{- $releasedIn := printf "%v" $feature.released_in -}} - {{- $releaseUrl := printf "https://github.com/apache/parquet-format/releases/tag/%s" $feature.release_tag -}} + {{- /* Fail the build if a dated feature points at an unknown parquet-format release. */ -}} + {{- $release := index $formatReleases $releasedIn -}} + {{- if not $release -}} + {{- errorf "missing parquet-format release metadata for %q" $releasedIn -}} + {{- end -}} + {{- $releaseUrl := printf "https://github.com/apache/parquet-format/releases/tag/%s" $release.tag -}} {{- if $featureUrl -}} @@ -69,7 +76,7 @@ {{ $releasedIn }} - {{ $feature.release_display }} + {{ $release.source_display }} {{- with $feature.approval -}} diff --git a/layouts/shortcodes/implementation-status.html b/layouts/shortcodes/implementation-status.html index d8c53050..b2c9742b 100644 --- a/layouts/shortcodes/implementation-status.html +++ b/layouts/shortcodes/implementation-status.html @@ -7,6 +7,8 @@ {{- $engines := site.Data.implementations.engines -}} {{- $categories := site.Data.implementations.categories -}} {{- $legend := site.Data.implementations.legend -}} +{{- /* Shared release-tag metadata keyed by feature released_in version. */ -}} +{{- $formatReleases := index site.Data.implementations "parquet-format-releases" -}} {{- /* Render each category */ -}} @@ -78,7 +80,13 @@

{{ $category.name }} {{- else -}} {{- $year = substr $dateStr 0 4 -}} {{- end -}} - {{- $releaseUrl := printf "https://github.com/apache/parquet-format/releases/tag/%s" .release_tag -}} + {{- $releasedIn := printf "%v" .released_in -}} + {{- /* Fail the build if a dated feature points at an unknown parquet-format release. */ -}} + {{- $release := index $formatReleases $releasedIn -}} + {{- if not $release -}} + {{- errorf "missing parquet-format release metadata for %q" $releasedIn -}} + {{- end -}} + {{- $releaseUrl := printf "https://github.com/apache/parquet-format/releases/tag/%s" $release.tag -}} ({{ $year }}) {{- end -}} From 9b1c6b5443daa65de57ada2e41723f7b5f5a5a7a Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 26 Jun 2026 13:27:57 -0400 Subject: [PATCH 09/11] simplify comments --- data/implementations/parquet-format-releases.yaml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/data/implementations/parquet-format-releases.yaml b/data/implementations/parquet-format-releases.yaml index 30a62810..b6c4dd46 100644 --- a/data/implementations/parquet-format-releases.yaml +++ b/data/implementations/parquet-format-releases.yaml @@ -1,18 +1,7 @@ -# Metadata about releases of the apache/parquet-format repository. +# Metadata about apache/parquet-format repository releases # # Each top-level key is a parquet-format version. The values describe the # GitHub release tag and the source tree or compare link for that release. -# -# This is intentionally separate from data/implementations/features/*.yaml: -# many feature rows were released in the same parquet-format version, so they -# can reference the shared metadata with only their `released_in` value. -# -# The shortcodes use this file to build "Released in" and "Source" links in the -# implementation and format-version tables. -# -# Older releases used the `parquet-format-*` tag prefix. Newer releases use -# `apache-parquet-format-*`, so this map keeps that historical detail out of -# each individual feature row. 1.0.0: tag: parquet-format-1.0.0 source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 From c0258a85e36f88f25f2aef137c4e1975042d716f Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 26 Jun 2026 13:35:19 -0400 Subject: [PATCH 10/11] restore notes --- layouts/shortcodes/format-versions.html | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/layouts/shortcodes/format-versions.html b/layouts/shortcodes/format-versions.html index 5279d5ad..21e30f6e 100644 --- a/layouts/shortcodes/format-versions.html +++ b/layouts/shortcodes/format-versions.html @@ -50,6 +50,7 @@ Released in Source Approved + Notes @@ -83,6 +84,11 @@ {{ . | markdownify }} {{- end -}} + + {{- with $feature.version_note -}} + {{ . | markdownify }} + {{- end -}} + {{- end }} From 9f8d58a2087eac197bcb54adf5b5ef013aad3ca7 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 26 Jun 2026 13:38:27 -0400 Subject: [PATCH 11/11] Quote release versions --- .../features/compressions.yaml | 16 +++--- data/implementations/features/encodings.yaml | 20 ++++---- .../features/format-features.yaml | 22 ++++---- .../features/logical-types.yaml | 50 +++++++++---------- .../features/physical-types.yaml | 16 +++--- .../parquet-format-releases.yaml | 26 +++++----- 6 files changed, 75 insertions(+), 75 deletions(-) diff --git a/data/implementations/features/compressions.yaml b/data/implementations/features/compressions.yaml index 26a568cb..8165683b 100644 --- a/data/implementations/features/compressions.yaml +++ b/data/implementations/features/compressions.yaml @@ -3,40 +3,40 @@ features: - id: compression-uncompressed display_name: UNCOMPRESSED compatibility: forward_incompatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#uncompressed - id: compression-brotli display_name: BROTLI compatibility: forward_incompatible - released_in: 2.4.0 + released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#brotli - id: compression-gzip display_name: GZIP compatibility: forward_incompatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#gzip - id: compression-lz4-deprecated display_name: LZ4 (deprecated) compatibility: forward_incompatible - released_in: 2.4.0 + released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lz4 - id: compression-lz4-raw display_name: LZ4_RAW compatibility: forward_incompatible - released_in: 2.9.0 + released_in: "2.9.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lz4_raw - id: compression-lzo display_name: LZO compatibility: forward_incompatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lzo - id: compression-snappy display_name: SNAPPY compatibility: forward_incompatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#snappy - id: compression-zstd display_name: ZSTD compatibility: forward_incompatible - released_in: 2.4.0 + released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#zstd diff --git a/data/implementations/features/encodings.yaml b/data/implementations/features/encodings.yaml index 9c478095..e03eed38 100644 --- a/data/implementations/features/encodings.yaml +++ b/data/implementations/features/encodings.yaml @@ -3,47 +3,47 @@ features: - id: encoding-plain display_name: PLAIN compatibility: forward_incompatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#plain-plain--0 - id: encoding-plain-dictionary display_name: PLAIN_DICTIONARY compatibility: forward_incompatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 - id: encoding-rle-dictionary display_name: RLE_DICTIONARY compatibility: forward_incompatible - released_in: 2.0.0 + released_in: "2.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 - id: encoding-rle display_name: RLE compatibility: forward_incompatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3 - id: encoding-bit-packed display_name: BIT_PACKED (deprecated) compatibility: forward_incompatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#bit-packed-deprecated-bit_packed--4 - id: encoding-delta-binary-packed display_name: DELTA_BINARY_PACKED compatibility: forward_incompatible - released_in: 2.0.0 + released_in: "2.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-encoding-delta_binary_packed--5 - id: encoding-delta-length-byte-array display_name: DELTA_LENGTH_BYTE_ARRAY compatibility: forward_incompatible - released_in: 2.0.0 + released_in: "2.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-length-byte-array-delta_length_byte_array--6 - id: encoding-delta-byte-array display_name: DELTA_BYTE_ARRAY compatibility: forward_incompatible - released_in: 2.0.0 + released_in: "2.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-strings-delta_byte_array--7 - id: encoding-byte-stream-split display_name: BYTE_STREAM_SPLIT compatibility: forward_incompatible - released_in: 2.8.0 + released_in: "2.8.0" release_date: '2020-01-13' feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 approval: "[2019-12-03](https://lists.apache.org/thread/xs5qt2odm299pxgqb22mty2csc1so5yr)" @@ -52,7 +52,7 @@ features: spec_url: https://github.com/apache/parquet-format/commit/e517ac4dbe08d518eb5c2e58576d4c711973db94 version_display_name: BYTE_STREAM_SPLIT
(Additional Types) compatibility: forward_incompatible - released_in: 2.11.0 + released_in: "2.11.0" release_date: '2024-03-21' feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 approval: "[2024-03-18](https://lists.apache.org/thread/nlsj0ftxy7y4ov1678rgy5zc7dmogg6q)" diff --git a/data/implementations/features/format-features.yaml b/data/implementations/features/format-features.yaml index 0008d1b1..b45a5847 100644 --- a/data/implementations/features/format-features.yaml +++ b/data/implementations/features/format-features.yaml @@ -4,7 +4,7 @@ features: display_name: xxHash-based bloom filters spec_url: https://github.com/apache/parquet-format/blob/master/BloomFilter.md compatibility: forward_compatible - released_in: 2.7.0 + released_in: "2.7.0" release_date: '2019-09-25' feature_url: https://github.com/apache/parquet-format/blob/master/BloomFilter.md approval: "[2019-09-09](https://lists.apache.org/thread/ktdx1xp0d2gjfgkcvd29zxvt3cgg88bo)" @@ -12,7 +12,7 @@ features: display_name: Bloom filter length note: 'In parquet.thrift: ColumnMetaData->bloom_filter_length' compatibility: forward_compatible - released_in: 2.10.0 + released_in: "2.10.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L933 - id: format-stats-min-max display_name: Statistics min_value, max_value @@ -20,19 +20,19 @@ features: display_name: Page index spec_url: https://github.com/apache/parquet-format/blob/master/PageIndex.md compatibility: forward_compatible - released_in: 2.4.0 + released_in: "2.4.0" release_date: '2018-05-04' feature_url: https://github.com/apache/parquet-format/blob/master/PageIndex.md - id: format-page-crc32 display_name: Page CRC32 checksum compatibility: forward_compatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L829 - id: format-modular-encryption display_name: Modular encryption spec_url: https://github.com/apache/parquet-format/blob/master/Encryption.md compatibility: forward_incompatible - released_in: 2.7.0 + released_in: "2.7.0" release_date: '2019-09-25' feature_url: https://github.com/apache/parquet-format/blob/master/Encryption.md approval: "[2019-01-16](https://lists.apache.org/thread/l8zcwnbrnhjh3w2k1lyb0v6ct5lnzr0h)" @@ -40,7 +40,7 @@ features: display_name: Size statistics note: 'In parquet.thrift: ColumnMetaData->size_statistics' compatibility: forward_compatible - released_in: 2.10.0 + released_in: "2.10.0" release_date: '2023-11-20' feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L202 approval: "[2023-11-14](https://lists.apache.org/thread/wgobz41mfldbhqpg9q4mdwypghg2cxg2)" @@ -48,32 +48,32 @@ features: display_name: Data Page V2 note: 'In parquet.thrift: DataPageHeaderV2' compatibility: forward_incompatible - released_in: 2.0.0 + released_in: "2.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L724 - id: format-data-page-v1 display_name: Data Page V1 implementation_status: false compatibility: forward_incompatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L671 - id: format-geospatial-statistics display_name: Geospatial statistics implementation_status: false compatibility: forward_compatible - released_in: 2.11.0 + released_in: "2.11.0" feature_url: https://github.com/apache/parquet-format/blob/master/Geospatial.md#statistics approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: format-binary-protocol-extensions display_name: Binary protocol extensions implementation_status: false compatibility: forward_compatible - released_in: 2.11.0 + released_in: "2.11.0" feature_url: https://github.com/apache/parquet-format/blob/master/BinaryProtocolExtensions.md approval: "[2024-09-06](https://lists.apache.org/thread/x3472kldrq5kjnld9ztj1jozz25f40hg)" - id: format-ieee-754-total-order-and-nan-counts display_name: IEEE 754 total order and NaN counts implementation_status: false compatibility: forward_compatible - released_in: 2.13.0 + released_in: "2.13.0" feature_url: https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift#L1061 approval: "[2026-05-26](https://lists.apache.org/thread/h0k0hqo0sojqphnbnrkp8b0gmwdzq9on)" diff --git a/data/implementations/features/logical-types.yaml b/data/implementations/features/logical-types.yaml index 3cfe5490..9adf65dc 100644 --- a/data/implementations/features/logical-types.yaml +++ b/data/implementations/features/logical-types.yaml @@ -4,51 +4,51 @@ features: display_name: STRING version_display_name: STRING (BYTE_ARRAY) compatibility: forward_compatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#string - id: logical-enum display_name: ENUM version_display_name: ENUM (BYTE_ARRAY) compatibility: forward_compatible - released_in: 2.0.0 + released_in: "2.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#enum - id: logical-uuid display_name: UUID version_display_name: UUID (FIXED_LEN_BYTE_ARRAY(16)) compatibility: forward_compatible - released_in: 2.6.0 + released_in: "2.6.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#uuid - id: logical-int-types display_name: 8, 16, 32, 64 bit signed and unsigned INT version_display_name: Signed and unsigned integer logical types (INT32, INT64) compatibility: forward_compatible - released_in: 2.2.0 + released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#signed-integers - id: logical-decimal-int32 display_name: DECIMAL (INT32) compatibility: forward_compatible - released_in: 2.1.0 + released_in: "2.1.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal - id: logical-decimal-int64 display_name: DECIMAL (INT64) compatibility: forward_compatible - released_in: 2.1.0 + released_in: "2.1.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal - id: logical-decimal-byte-array display_name: DECIMAL (BYTE_ARRAY) compatibility: forward_compatible - released_in: 2.1.0 + released_in: "2.1.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal - id: logical-decimal-fixed-len-byte-array display_name: DECIMAL (FIXED_LEN_BYTE_ARRAY) compatibility: forward_compatible - released_in: 2.1.0 + released_in: "2.1.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal - id: logical-float16 display_name: FLOAT16 version_display_name: FLOAT16 (FIXED_LEN_BYTE_ARRAY(2)) compatibility: forward_compatible - released_in: 2.10.0 + released_in: "2.10.0" release_date: '2023-11-20' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#float16 approval: "[2023-10-13](https://lists.apache.org/thread/gyvqcx9ssxkjlrwogqwy7n4z6ofdm871)" @@ -56,46 +56,46 @@ features: display_name: DATE version_display_name: DATE (INT32) compatibility: forward_compatible - released_in: 2.2.0 + released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#date - id: logical-time-int32 display_name: TIME (INT32) compatibility: forward_compatible - released_in: 2.2.0 + released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time - id: logical-time-int64 display_name: TIME (INT64) compatibility: forward_compatible - released_in: 2.4.0 + released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time - id: logical-timestamp-int64 display_name: TIMESTAMP (INT64) compatibility: forward_compatible - released_in: 2.2.0 + released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#timestamp - id: logical-interval display_name: INTERVAL version_display_name: INTERVAL (FIXED_LEN_BYTE_ARRAY(12)) compatibility: forward_compatible - released_in: 2.2.0 + released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#interval - id: logical-json display_name: JSON version_display_name: JSON (BYTE_ARRAY) compatibility: forward_compatible - released_in: 2.2.0 + released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#json - id: logical-bson display_name: BSON version_display_name: BSON (BYTE_ARRAY) compatibility: forward_compatible - released_in: 2.2.0 + released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#bson - id: logical-variant display_name: VARIANT spec_url: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md compatibility: forward_compatible - released_in: 2.12.0 + released_in: "2.12.0" release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md approval: "[2025-08-24](https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7)" @@ -104,7 +104,7 @@ features: spec_url: https://github.com/apache/parquet-format/blob/master/Geospatial.md#logical-types version_display_name: GEOMETRY (BYTE_ARRAY) compatibility: forward_compatible - released_in: 2.11.0 + released_in: "2.11.0" release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geometry approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" @@ -113,42 +113,42 @@ features: spec_url: https://github.com/apache/parquet-format/blob/master/Geospatial.md#logical-types version_display_name: GEOGRAPHY (BYTE_ARRAY) compatibility: forward_compatible - released_in: 2.11.0 + released_in: "2.11.0" release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geography approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: logical-list display_name: LIST compatibility: forward_compatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists - id: logical-map display_name: MAP compatibility: forward_compatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#maps - id: logical-unknown display_name: UNKNOWN (always null) compatibility: forward_compatible - released_in: 2.4.0 + released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#unknown-always-null - id: logical-type-union display_name: LogicalType union implementation_status: false compatibility: forward_compatible - released_in: 2.4.0 + released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L471 version_note: Supersedes `ConvertedType` enum
deprecated in [2.9.0](https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.9.0) - id: logical-nanosecond-time-timestamp display_name: Nanosecond TIME/TIMESTAMP implementation_status: false compatibility: forward_compatible - released_in: 2.6.0 + released_in: "2.6.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L352 - id: logical-variant-shredding display_name: Variant shredding implementation_status: false compatibility: forward_compatible - released_in: 2.12.0 + released_in: "2.12.0" feature_url: https://github.com/apache/parquet-format/blob/master/VariantShredding.md approval: "[2025-08-24](https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7)" diff --git a/data/implementations/features/physical-types.yaml b/data/implementations/features/physical-types.yaml index 2f0dddfd..3c0b1bcb 100644 --- a/data/implementations/features/physical-types.yaml +++ b/data/implementations/features/physical-types.yaml @@ -3,42 +3,42 @@ features: - id: physical-boolean display_name: BOOLEAN compatibility: forward_incompatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L33 - id: physical-int32 display_name: INT32 compatibility: forward_incompatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L34 - id: physical-int64 display_name: INT64 compatibility: forward_incompatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L35 - id: physical-int96 display_name: INT96 note: This type is deprecated, but as of 2024 it's common in currently produced parquet files version_display_name: INT96 (deprecated) compatibility: forward_incompatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L36 - id: physical-float display_name: FLOAT compatibility: forward_incompatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L37 - id: physical-double display_name: DOUBLE compatibility: forward_incompatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L38 - id: physical-byte-array display_name: BYTE_ARRAY compatibility: forward_incompatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L39 - id: physical-fixed-len-byte-array display_name: FIXED_LEN_BYTE_ARRAY compatibility: forward_incompatible - released_in: 1.0.0 + released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L40 diff --git a/data/implementations/parquet-format-releases.yaml b/data/implementations/parquet-format-releases.yaml index b6c4dd46..d41445c8 100644 --- a/data/implementations/parquet-format-releases.yaml +++ b/data/implementations/parquet-format-releases.yaml @@ -2,55 +2,55 @@ # # Each top-level key is a parquet-format version. The values describe the # GitHub release tag and the source tree or compare link for that release. -1.0.0: +"1.0.0": tag: parquet-format-1.0.0 source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 source_display: 1.0.0 -2.0.0: +"2.0.0": tag: parquet-format-2.0.0 source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 source_display: 1.0.0..2.0.0 -2.1.0: +"2.1.0": tag: parquet-format-2.1.0 source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.0.0...parquet-format-2.1.0 source_display: 2.0.0..2.1.0 -2.2.0: +"2.2.0": tag: apache-parquet-format-2.2.0 source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 source_display: 2.1.0..2.2.0 -2.4.0: +"2.4.0": tag: apache-parquet-format-2.4.0 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 source_display: 2.3.1..2.4.0 -2.6.0: +"2.6.0": tag: apache-parquet-format-2.6.0 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.5.0...apache-parquet-format-2.6.0 source_display: 2.5.0..2.6.0 -2.7.0: +"2.7.0": tag: apache-parquet-format-2.7.0 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.6.0...apache-parquet-format-2.7.0 source_display: 2.6.0..2.7.0 -2.8.0: +"2.8.0": tag: apache-parquet-format-2.8.0 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.7.0...apache-parquet-format-2.8.0 source_display: 2.7.0..2.8.0 -2.9.0: +"2.9.0": tag: apache-parquet-format-2.9.0 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.8.0...apache-parquet-format-2.9.0 source_display: 2.8.0..2.9.0 -2.10.0: +"2.10.0": tag: apache-parquet-format-2.10.0 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.9.0...apache-parquet-format-2.10.0 source_display: 2.9.0..2.10.0 -2.11.0: +"2.11.0": tag: apache-parquet-format-2.11.0 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 source_display: 2.10.0..2.11.0 -2.12.0: +"2.12.0": tag: apache-parquet-format-2.12.0 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.11.0...apache-parquet-format-2.12.0 source_display: 2.11.0..2.12.0 -2.13.0: +"2.13.0": tag: apache-parquet-format-2.13.0 source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.12.0...apache-parquet-format-2.13.0 source_display: 2.12.0..2.13.0