diff --git a/content/en/docs/File Format/implementationstatus.md b/content/en/docs/File Format/implementationstatus.md index 47e7c22c..96cffb89 100644 --- a/content/en/docs/File Format/implementationstatus.md +++ b/content/en/docs/File Format/implementationstatus.md @@ -28,5 +28,5 @@ The value in each box means: * [duckdb](https://github.com/duckdb/duckdb) (C++) * [polars](https://github.com/pola-rs/polars) (Rust) - + {{< implementation-status >}} diff --git a/content/en/docs/File Format/versions.md b/content/en/docs/File Format/versions.md index 3f666d22..00b7f0df 100644 --- a/content/en/docs/File Format/versions.md +++ b/content/en/docs/File Format/versions.md @@ -67,43 +67,15 @@ the next parquet-format release. [contributing guide]: https://github.com/apache/parquet-format/blob/master/CONTRIBUTING.md#additionschanges-to-the-format [note in parquet.thrift]: https://github.com/apache/parquet-format/blob/74001e41f5c5a1856b29be115f9c992cab16a4bf/src/main/thrift/parquet.thrift#L1368-L1373 [closing-out-2.0]: https://lists.apache.org/thread/0bdyyb7qobrxx94x8v7t5z7g2ksnpyr2 +[encrypted footer]: https://github.com/apache/parquet-format/blob/master/Encryption.md#54-encrypted-footer-mode +[plaintext footer]: https://github.com/apache/parquet-format/blob/master/Encryption.md#55-plaintext-footer-mode ## Forward incompatible features by version Forward incompatible features and the format version each became available in: -| Feature | Released in | Source | Notes | -| ------------------------------------------ | ----------------------------- | --- | ------------------------- | -| [BOOLEAN] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [INT32] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [INT64] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [INT96 (deprecated)] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [FLOAT] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [DOUBLE] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [BYTE_ARRAY] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [FIXED_LEN_BYTE_ARRAY] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [Data Page V1] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [Data Page V2] | [2.0.0] | [1.0.0..2.0.0] | | -| [PLAIN] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [PLAIN_DICTIONARY] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [RLE] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [BIT_PACKED (deprecated)] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [RLE_DICTIONARY] | [2.0.0] | [1.0.0..2.0.0] | | -| [DELTA_BINARY_PACKED] | [2.0.0] | [1.0.0..2.0.0] | | -| [DELTA_LENGTH_BYTE_ARRAY] | [2.0.0] | [1.0.0..2.0.0] | | -| [DELTA_BYTE_ARRAY] | [2.0.0] | [1.0.0..2.0.0] | | -| [BYTE_STREAM_SPLIT] | [2.8.0] | [2.7.0..2.8.0] | [Approved 2019-12-03] | -| [BYTE_STREAM_SPLIT
(Additional Types)] | [2.11.0] | [2.10.0..2.11.0] | [Approved 2024-03-18] | -| [UNCOMPRESSED] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [SNAPPY] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [GZIP] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [LZO] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [BROTLI] | [2.4.0] | [2.3.1..2.4.0] | | -| [LZ4 (deprecated)] | [2.4.0] | [2.3.1..2.4.0] | | -| [LZ4_RAW] | [2.9.0] | [2.8.0..2.9.0] | | -| [ZSTD] | [2.4.0] | [2.3.1..2.4.0] | | -| [Modular encryption] | [2.7.0] | [2.6.0..2.7.0] | [Approved 2019-01-16] | - + +{{< format-versions table="forward_incompatible" >}} > **Note:** Files with an [encrypted footer] use different magic bytes (`PARE` > instead of `PAR1`), making it clear to readers they must support modular @@ -115,147 +87,5 @@ Forward incompatible features and the format version each became available in: Older readers can read files that use these features but may not understand the new information. -| Feature | Released in | Source | Notes | -| ------------------------------------------- | ----------------------------- | --- |-----------------------------------------------------------| -| [xxHash-based bloom filters] | [2.7.0] | [2.6.0..2.7.0] | [Approved 2019-09-09] | -| [Bloom filter length] | [2.10.0] | [2.9.0..2.10.0] | | -| [Page index] | [2.4.0] | [2.3.1..2.4.0] | | -| [Page CRC32 checksum] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [Size statistics] | [2.10.0] | [2.9.0..2.10.0] | [Approved 2023-11-14] | -| [Geospatial statistics] | [2.11.0] | [2.10.0..2.11.0] | [Approved 2025-02-09] | -| [Binary protocol extensions] | [2.11.0] | [2.10.0..2.11.0] | [Approved 2024-09-06] | -| [IEEE 754 total order and NaN counts] | [2.13.0] | [2.12.0..2.13.0] | [Approved 2026-05-26] | -| [LogicalType union] | [2.4.0] | [2.3.1..2.4.0] | Supersedes `ConvertedType` enum
deprecated in [2.9.0] | -| [STRING (BYTE_ARRAY)] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [ENUM (BYTE_ARRAY)] | [2.0.0] | [1.0.0..2.0.0] | | -| [UUID (FIXED_LEN_BYTE_ARRAY(16))] | [2.6.0] | [2.5.0..2.6.0] | | -| [Signed and unsigned integer logical types (INT32, INT64)] | [2.2.0] | [2.1.0..2.2.0] | | -| [DECIMAL (INT32)] | [2.1.0] | [2.0.0..2.1.0] | | -| [DECIMAL (INT64)] | [2.1.0] | [2.0.0..2.1.0] | | -| [DECIMAL (BYTE_ARRAY)] | [2.1.0] | [2.0.0..2.1.0] | | -| [DECIMAL (FIXED_LEN_BYTE_ARRAY)] | [2.1.0] | [2.0.0..2.1.0] | | -| [FLOAT16 (FIXED_LEN_BYTE_ARRAY(2))] | [2.10.0] | [2.9.0..2.10.0] | [Approved 2023-10-13] | -| [DATE (INT32)] | [2.2.0] | [2.1.0..2.2.0] | | -| [TIME (INT32)] | [2.2.0] | [2.1.0..2.2.0] | | -| [TIME (INT64)] | [2.4.0] | [2.3.1..2.4.0] | | -| [TIMESTAMP (INT64)] | [2.2.0] | [2.1.0..2.2.0] | | -| [Nanosecond TIME/TIMESTAMP] | [2.6.0] | [2.5.0..2.6.0] | | -| [INTERVAL (FIXED_LEN_BYTE_ARRAY(12))] | [2.2.0] | [2.1.0..2.2.0] | | -| [JSON (BYTE_ARRAY)] | [2.2.0] | [2.1.0..2.2.0] | | -| [BSON (BYTE_ARRAY)] | [2.2.0] | [2.1.0..2.2.0] | | -| [VARIANT] | [2.12.0] | [2.11.0..2.12.0] | [Approved 2025-08-24] | -| [Variant shredding] | [2.12.0] | [2.11.0..2.12.0] | [Approved 2025-08-24] | -| [GEOMETRY (BYTE_ARRAY)] | [2.11.0] | [2.10.0..2.11.0] | [Approved 2025-02-09] | -| [GEOGRAPHY (BYTE_ARRAY)] | [2.11.0] | [2.10.0..2.11.0] | [Approved 2025-02-09] | -| [LIST] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [MAP] | [1.0.0] | [1.0.0][tree-1.0.0] | | -| [UNKNOWN (always null)] | [2.4.0] | [2.3.1..2.4.0] | | - -[PLAIN]: https://github.com/apache/parquet-format/blob/master/Encodings.md#plain-plain--0 -[PLAIN_DICTIONARY]: https://github.com/apache/parquet-format/blob/master/Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 -[RLE]: https://github.com/apache/parquet-format/blob/master/Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3 -[RLE_DICTIONARY]: https://github.com/apache/parquet-format/blob/master/Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 -[DELTA_BINARY_PACKED]: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-encoding-delta_binary_packed--5 -[DELTA_LENGTH_BYTE_ARRAY]: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-length-byte-array-delta_length_byte_array--6 -[DELTA_BYTE_ARRAY]: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-strings-delta_byte_array--7 -[BYTE_STREAM_SPLIT]: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 -[Modular encryption]: https://github.com/apache/parquet-format/blob/master/Encryption.md -[encrypted footer]: https://github.com/apache/parquet-format/blob/master/Encryption.md#54-encrypted-footer-mode -[plaintext footer]: https://github.com/apache/parquet-format/blob/master/Encryption.md#55-plaintext-footer-mode -[xxHash-based bloom filters]: https://github.com/apache/parquet-format/blob/master/BloomFilter.md -[Page index]: https://github.com/apache/parquet-format/blob/master/PageIndex.md -[FLOAT16 (FIXED_LEN_BYTE_ARRAY(2))]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#float16 -[VARIANT]: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md -[GEOMETRY (BYTE_ARRAY)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geometry -[GEOGRAPHY (BYTE_ARRAY)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geography - -[1.0.0]: https://github.com/apache/parquet-format/releases/tag/parquet-format-1.0.0 -[2.0.0]: https://github.com/apache/parquet-format/releases/tag/parquet-format-2.0.0 -[2.8.0]: https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.8.0 -[2.11.0]: https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.11.0 -[2.4.0]: https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.4.0 -[2.9.0]: https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.9.0 -[2.7.0]: https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.7.0 -[2.10.0]: https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.10.0 -[2.12.0]: https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.12.0 -[2.13.0]: https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.13.0 - -[Approved 2019-12-03]: https://lists.apache.org/thread/xs5qt2odm299pxgqb22mty2csc1so5yr -[Approved 2024-03-18]: https://lists.apache.org/thread/nlsj0ftxy7y4ov1678rgy5zc7dmogg6q -[Approved 2019-01-16]: https://lists.apache.org/thread/l8zcwnbrnhjh3w2k1lyb0v6ct5lnzr0h -[Approved 2019-09-09]: https://lists.apache.org/thread/ktdx1xp0d2gjfgkcvd29zxvt3cgg88bo -[Approved 2023-11-14]: https://lists.apache.org/thread/wgobz41mfldbhqpg9q4mdwypghg2cxg2 -[Approved 2023-10-13]: https://lists.apache.org/thread/gyvqcx9ssxkjlrwogqwy7n4z6ofdm871 -[Approved 2025-08-24]: https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7 -[Approved 2025-02-09]: https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo - -[1.0.0..2.0.0]: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 -[2.7.0..2.8.0]: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.7.0...apache-parquet-format-2.8.0 -[2.10.0..2.11.0]: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 -[2.3.1..2.4.0]: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 -[2.8.0..2.9.0]: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.8.0...apache-parquet-format-2.9.0 -[2.6.0..2.7.0]: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.6.0...apache-parquet-format-2.7.0 -[2.9.0..2.10.0]: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.9.0...apache-parquet-format-2.10.0 -[2.11.0..2.12.0]: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.11.0...apache-parquet-format-2.12.0 -[2.12.0..2.13.0]: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.12.0...apache-parquet-format-2.13.0 - -[2.1.0]: https://github.com/apache/parquet-format/releases/tag/parquet-format-2.1.0 -[2.2.0]: https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.2.0 -[2.6.0]: https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.6.0 -[2.0.0..2.1.0]: https://github.com/apache/parquet-format/compare/parquet-format-2.0.0...parquet-format-2.1.0 -[2.1.0..2.2.0]: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 -[2.5.0..2.6.0]: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.5.0...apache-parquet-format-2.6.0 - -[tree-1.0.0]: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 - -[Variant shredding]: https://github.com/apache/parquet-format/blob/master/VariantShredding.md -[Geospatial statistics]: https://github.com/apache/parquet-format/blob/master/Geospatial.md#statistics -[Binary protocol extensions]: https://github.com/apache/parquet-format/blob/master/BinaryProtocolExtensions.md -[Approved 2024-09-06]: https://lists.apache.org/thread/x3472kldrq5kjnld9ztj1jozz25f40hg -[Approved 2026-05-26]: https://lists.apache.org/thread/h0k0hqo0sojqphnbnrkp8b0gmwdzq9on - -[Bloom filter length]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L933 -[Page CRC32 checksum]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L829 -[Size statistics]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L202 -[LogicalType union]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L471 -[Nanosecond TIME/TIMESTAMP]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L352 -[STRING (BYTE_ARRAY)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#string -[ENUM (BYTE_ARRAY)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#enum -[UUID (FIXED_LEN_BYTE_ARRAY(16))]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#uuid -[DECIMAL (INT32)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal -[DECIMAL (INT64)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal -[DECIMAL (BYTE_ARRAY)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal -[DECIMAL (FIXED_LEN_BYTE_ARRAY)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal -[DATE (INT32)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#date -[TIME (INT32)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time -[TIME (INT64)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time -[TIMESTAMP (INT64)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#timestamp -[INTERVAL (FIXED_LEN_BYTE_ARRAY(12))]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#interval -[JSON (BYTE_ARRAY)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#json -[BSON (BYTE_ARRAY)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#bson -[LIST]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists -[MAP]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#maps -[UNKNOWN (always null)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#unknown-always-null -[Signed and unsigned integer logical types (INT32, INT64)]: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#signed-integers -[IEEE 754 total order and NaN counts]: https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift#L1061 - -[BOOLEAN]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L33 -[INT32]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L34 -[INT64]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L35 -[INT96 (deprecated)]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L36 -[FLOAT]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L37 -[DOUBLE]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L38 -[BYTE_ARRAY]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L39 -[FIXED_LEN_BYTE_ARRAY]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L40 -[Data Page V1]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L671 -[Data Page V2]: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L724 -[BIT_PACKED (deprecated)]: https://github.com/apache/parquet-format/blob/master/Encodings.md#bit-packed-deprecated-bit_packed--4 -[BYTE_STREAM_SPLIT
(Additional Types)]: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 -[UNCOMPRESSED]: https://github.com/apache/parquet-format/blob/master/Compression.md#uncompressed -[SNAPPY]: https://github.com/apache/parquet-format/blob/master/Compression.md#snappy -[GZIP]: https://github.com/apache/parquet-format/blob/master/Compression.md#gzip -[LZO]: https://github.com/apache/parquet-format/blob/master/Compression.md#lzo -[BROTLI]: https://github.com/apache/parquet-format/blob/master/Compression.md#brotli -[LZ4 (deprecated)]: https://github.com/apache/parquet-format/blob/master/Compression.md#lz4 -[LZ4_RAW]: https://github.com/apache/parquet-format/blob/master/Compression.md#lz4_raw -[ZSTD]: https://github.com/apache/parquet-format/blob/master/Compression.md#zstd + +{{< format-versions table="forward_compatible" >}} diff --git a/data/implementations/features/compressions.yaml b/data/implementations/features/compressions.yaml index fec73a0c..8165683b 100644 --- a/data/implementations/features/compressions.yaml +++ b/data/implementations/features/compressions.yaml @@ -2,24 +2,41 @@ category_id: compressions features: - id: compression-uncompressed display_name: UNCOMPRESSED - + compatibility: forward_incompatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#uncompressed - id: compression-brotli display_name: BROTLI - + compatibility: forward_incompatible + released_in: "2.4.0" + feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#brotli - id: compression-gzip display_name: GZIP - + compatibility: forward_incompatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#gzip - id: compression-lz4-deprecated display_name: LZ4 (deprecated) - + compatibility: forward_incompatible + released_in: "2.4.0" + feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lz4 - id: compression-lz4-raw display_name: LZ4_RAW - + compatibility: forward_incompatible + released_in: "2.9.0" + feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lz4_raw - id: compression-lzo display_name: LZO - + compatibility: forward_incompatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lzo - id: compression-snappy display_name: SNAPPY - + compatibility: forward_incompatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#snappy - id: compression-zstd display_name: ZSTD + compatibility: forward_incompatible + released_in: "2.4.0" + feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#zstd diff --git a/data/implementations/features/encodings.yaml b/data/implementations/features/encodings.yaml index 03614d9b..e03eed38 100644 --- a/data/implementations/features/encodings.yaml +++ b/data/implementations/features/encodings.yaml @@ -2,37 +2,57 @@ category_id: encodings features: - id: encoding-plain display_name: PLAIN - + compatibility: forward_incompatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#plain-plain--0 - id: encoding-plain-dictionary display_name: PLAIN_DICTIONARY - + compatibility: forward_incompatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 - id: encoding-rle-dictionary display_name: RLE_DICTIONARY - + compatibility: forward_incompatible + released_in: "2.0.0" + feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 - id: encoding-rle display_name: RLE - + compatibility: forward_incompatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3 - id: encoding-bit-packed display_name: BIT_PACKED (deprecated) - + compatibility: forward_incompatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#bit-packed-deprecated-bit_packed--4 - id: encoding-delta-binary-packed display_name: DELTA_BINARY_PACKED - + compatibility: forward_incompatible + released_in: "2.0.0" + feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-encoding-delta_binary_packed--5 - id: encoding-delta-length-byte-array display_name: DELTA_LENGTH_BYTE_ARRAY - + compatibility: forward_incompatible + released_in: "2.0.0" + feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-length-byte-array-delta_length_byte_array--6 - id: encoding-delta-byte-array display_name: DELTA_BYTE_ARRAY - + compatibility: forward_incompatible + released_in: "2.0.0" + feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-strings-delta_byte_array--7 - id: encoding-byte-stream-split display_name: BYTE_STREAM_SPLIT - format_version: - version: "2.8.0" - date: "2020-01-13" - + compatibility: forward_incompatible + released_in: "2.8.0" + release_date: '2020-01-13' + feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 + approval: "[2019-12-03](https://lists.apache.org/thread/xs5qt2odm299pxgqb22mty2csc1so5yr)" - id: encoding-byte-stream-split-extended display_name: BYTE_STREAM_SPLIT (Additional Types) spec_url: https://github.com/apache/parquet-format/commit/e517ac4dbe08d518eb5c2e58576d4c711973db94 - format_version: - version: "2.11.0" - date: "2024-03-21" + version_display_name: BYTE_STREAM_SPLIT
(Additional Types) + compatibility: forward_incompatible + released_in: "2.11.0" + release_date: '2024-03-21' + feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 + approval: "[2024-03-18](https://lists.apache.org/thread/nlsj0ftxy7y4ov1678rgy5zc7dmogg6q)" diff --git a/data/implementations/features/format-features.yaml b/data/implementations/features/format-features.yaml index 5a2dbfa4..b45a5847 100644 --- a/data/implementations/features/format-features.yaml +++ b/data/implementations/features/format-features.yaml @@ -3,41 +3,77 @@ features: - id: format-bloom-filters display_name: xxHash-based bloom filters spec_url: https://github.com/apache/parquet-format/blob/master/BloomFilter.md - format_version: - version: "2.7.0" - date: "2019-09-25" - + compatibility: forward_compatible + released_in: "2.7.0" + release_date: '2019-09-25' + feature_url: https://github.com/apache/parquet-format/blob/master/BloomFilter.md + approval: "[2019-09-09](https://lists.apache.org/thread/ktdx1xp0d2gjfgkcvd29zxvt3cgg88bo)" - id: format-bloom-filter-length display_name: Bloom filter length - note: "In parquet.thrift: ColumnMetaData->bloom_filter_length" - + note: 'In parquet.thrift: ColumnMetaData->bloom_filter_length' + compatibility: forward_compatible + released_in: "2.10.0" + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L933 - id: format-stats-min-max display_name: Statistics min_value, max_value - - id: format-page-index display_name: Page index spec_url: https://github.com/apache/parquet-format/blob/master/PageIndex.md - format_version: - version: "2.4.0" - date: "2018-05-04" - + compatibility: forward_compatible + released_in: "2.4.0" + release_date: '2018-05-04' + feature_url: https://github.com/apache/parquet-format/blob/master/PageIndex.md - id: format-page-crc32 display_name: Page CRC32 checksum - + compatibility: forward_compatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L829 - id: format-modular-encryption display_name: Modular encryption spec_url: https://github.com/apache/parquet-format/blob/master/Encryption.md - format_version: - version: "2.7.0" - date: "2019-09-25" - + compatibility: forward_incompatible + released_in: "2.7.0" + release_date: '2019-09-25' + feature_url: https://github.com/apache/parquet-format/blob/master/Encryption.md + approval: "[2019-01-16](https://lists.apache.org/thread/l8zcwnbrnhjh3w2k1lyb0v6ct5lnzr0h)" - id: format-size-statistics display_name: Size statistics - note: "In parquet.thrift: ColumnMetaData->size_statistics" - format_version: - version: "2.10.0" - date: "2023-11-20" - + note: 'In parquet.thrift: ColumnMetaData->size_statistics' + compatibility: forward_compatible + released_in: "2.10.0" + release_date: '2023-11-20' + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L202 + approval: "[2023-11-14](https://lists.apache.org/thread/wgobz41mfldbhqpg9q4mdwypghg2cxg2)" - id: format-data-page-v2 display_name: Data Page V2 - note: "In parquet.thrift: DataPageHeaderV2" + note: 'In parquet.thrift: DataPageHeaderV2' + compatibility: forward_incompatible + released_in: "2.0.0" + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L724 + - id: format-data-page-v1 + display_name: Data Page V1 + implementation_status: false + compatibility: forward_incompatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L671 + - id: format-geospatial-statistics + display_name: Geospatial statistics + implementation_status: false + compatibility: forward_compatible + released_in: "2.11.0" + feature_url: https://github.com/apache/parquet-format/blob/master/Geospatial.md#statistics + approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" + - id: format-binary-protocol-extensions + display_name: Binary protocol extensions + implementation_status: false + compatibility: forward_compatible + released_in: "2.11.0" + feature_url: https://github.com/apache/parquet-format/blob/master/BinaryProtocolExtensions.md + approval: "[2024-09-06](https://lists.apache.org/thread/x3472kldrq5kjnld9ztj1jozz25f40hg)" + - id: format-ieee-754-total-order-and-nan-counts + display_name: IEEE 754 total order and NaN counts + implementation_status: false + compatibility: forward_compatible + released_in: "2.13.0" + feature_url: https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift#L1061 + approval: "[2026-05-26](https://lists.apache.org/thread/h0k0hqo0sojqphnbnrkp8b0gmwdzq9on)" diff --git a/data/implementations/features/logical-types.yaml b/data/implementations/features/logical-types.yaml index 5d83b0bf..9adf65dc 100644 --- a/data/implementations/features/logical-types.yaml +++ b/data/implementations/features/logical-types.yaml @@ -2,81 +2,153 @@ category_id: logical-types features: - id: logical-string display_name: STRING - + version_display_name: STRING (BYTE_ARRAY) + compatibility: forward_compatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#string - id: logical-enum display_name: ENUM - + version_display_name: ENUM (BYTE_ARRAY) + compatibility: forward_compatible + released_in: "2.0.0" + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#enum - id: logical-uuid display_name: UUID - + version_display_name: UUID (FIXED_LEN_BYTE_ARRAY(16)) + compatibility: forward_compatible + released_in: "2.6.0" + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#uuid - id: logical-int-types display_name: 8, 16, 32, 64 bit signed and unsigned INT - + version_display_name: Signed and unsigned integer logical types (INT32, INT64) + compatibility: forward_compatible + released_in: "2.2.0" + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#signed-integers - id: logical-decimal-int32 display_name: DECIMAL (INT32) - + compatibility: forward_compatible + released_in: "2.1.0" + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal - id: logical-decimal-int64 display_name: DECIMAL (INT64) - + compatibility: forward_compatible + released_in: "2.1.0" + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal - id: logical-decimal-byte-array display_name: DECIMAL (BYTE_ARRAY) - + compatibility: forward_compatible + released_in: "2.1.0" + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal - id: logical-decimal-fixed-len-byte-array display_name: DECIMAL (FIXED_LEN_BYTE_ARRAY) - + compatibility: forward_compatible + released_in: "2.1.0" + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal - id: logical-float16 display_name: FLOAT16 - format_version: - version: "2.10.0" - date: "2023-11-20" - + version_display_name: FLOAT16 (FIXED_LEN_BYTE_ARRAY(2)) + compatibility: forward_compatible + released_in: "2.10.0" + release_date: '2023-11-20' + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#float16 + approval: "[2023-10-13](https://lists.apache.org/thread/gyvqcx9ssxkjlrwogqwy7n4z6ofdm871)" - id: logical-date display_name: DATE - + version_display_name: DATE (INT32) + compatibility: forward_compatible + released_in: "2.2.0" + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#date - id: logical-time-int32 display_name: TIME (INT32) - + compatibility: forward_compatible + released_in: "2.2.0" + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time - id: logical-time-int64 display_name: TIME (INT64) - + compatibility: forward_compatible + released_in: "2.4.0" + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time - id: logical-timestamp-int64 display_name: TIMESTAMP (INT64) - + compatibility: forward_compatible + released_in: "2.2.0" + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#timestamp - id: logical-interval display_name: INTERVAL - + version_display_name: INTERVAL (FIXED_LEN_BYTE_ARRAY(12)) + compatibility: forward_compatible + released_in: "2.2.0" + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#interval - id: logical-json display_name: JSON - + version_display_name: JSON (BYTE_ARRAY) + compatibility: forward_compatible + released_in: "2.2.0" + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#json - id: logical-bson display_name: BSON - + version_display_name: BSON (BYTE_ARRAY) + compatibility: forward_compatible + released_in: "2.2.0" + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#bson - id: logical-variant display_name: VARIANT spec_url: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md - format_version: - version: "2.11.0" - date: "2025-03-23" - + compatibility: forward_compatible + released_in: "2.12.0" + release_date: '2025-03-23' + feature_url: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md + approval: "[2025-08-24](https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7)" - id: logical-geometry display_name: GEOMETRY spec_url: https://github.com/apache/parquet-format/blob/master/Geospatial.md#logical-types - format_version: - version: "2.11.0" - date: "2025-03-23" - + version_display_name: GEOMETRY (BYTE_ARRAY) + compatibility: forward_compatible + released_in: "2.11.0" + release_date: '2025-03-23' + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geometry + approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: logical-geography display_name: GEOGRAPHY spec_url: https://github.com/apache/parquet-format/blob/master/Geospatial.md#logical-types - format_version: - version: "2.11.0" - date: "2025-03-23" - + version_display_name: GEOGRAPHY (BYTE_ARRAY) + compatibility: forward_compatible + released_in: "2.11.0" + release_date: '2025-03-23' + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geography + approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: logical-list display_name: LIST - + compatibility: forward_compatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists - id: logical-map display_name: MAP - + compatibility: forward_compatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#maps - id: logical-unknown display_name: UNKNOWN (always null) + compatibility: forward_compatible + released_in: "2.4.0" + feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#unknown-always-null + - id: logical-type-union + display_name: LogicalType union + implementation_status: false + compatibility: forward_compatible + released_in: "2.4.0" + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L471 + version_note: Supersedes `ConvertedType` enum
deprecated in [2.9.0](https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.9.0) + - id: logical-nanosecond-time-timestamp + display_name: Nanosecond TIME/TIMESTAMP + implementation_status: false + compatibility: forward_compatible + released_in: "2.6.0" + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L352 + - id: logical-variant-shredding + display_name: Variant shredding + implementation_status: false + compatibility: forward_compatible + released_in: "2.12.0" + feature_url: https://github.com/apache/parquet-format/blob/master/VariantShredding.md + approval: "[2025-08-24](https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7)" diff --git a/data/implementations/features/physical-types.yaml b/data/implementations/features/physical-types.yaml index 38d18711..3c0b1bcb 100644 --- a/data/implementations/features/physical-types.yaml +++ b/data/implementations/features/physical-types.yaml @@ -2,25 +2,43 @@ category_id: physical-types features: - id: physical-boolean display_name: BOOLEAN - + compatibility: forward_incompatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L33 - id: physical-int32 display_name: INT32 - + compatibility: forward_incompatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L34 - id: physical-int64 display_name: INT64 - + compatibility: forward_incompatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L35 - id: physical-int96 display_name: INT96 - note: "This type is deprecated, but as of 2024 it's common in currently produced parquet files" - + note: This type is deprecated, but as of 2024 it's common in currently produced parquet files + version_display_name: INT96 (deprecated) + compatibility: forward_incompatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L36 - id: physical-float display_name: FLOAT - + compatibility: forward_incompatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L37 - id: physical-double display_name: DOUBLE - + compatibility: forward_incompatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L38 - id: physical-byte-array display_name: BYTE_ARRAY - + compatibility: forward_incompatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L39 - id: physical-fixed-len-byte-array display_name: FIXED_LEN_BYTE_ARRAY + compatibility: forward_incompatible + released_in: "1.0.0" + feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L40 diff --git a/data/implementations/parquet-format-releases.yaml b/data/implementations/parquet-format-releases.yaml new file mode 100644 index 00000000..d41445c8 --- /dev/null +++ b/data/implementations/parquet-format-releases.yaml @@ -0,0 +1,56 @@ +# Metadata about apache/parquet-format repository releases +# +# Each top-level key is a parquet-format version. The values describe the +# GitHub release tag and the source tree or compare link for that release. +"1.0.0": + tag: parquet-format-1.0.0 + source_url: https://github.com/apache/parquet-format/tree/parquet-format-1.0.0 + source_display: 1.0.0 +"2.0.0": + tag: parquet-format-2.0.0 + source_url: https://github.com/apache/parquet-format/compare/parquet-format-1.0.0...parquet-format-2.0.0 + source_display: 1.0.0..2.0.0 +"2.1.0": + tag: parquet-format-2.1.0 + source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.0.0...parquet-format-2.1.0 + source_display: 2.0.0..2.1.0 +"2.2.0": + tag: apache-parquet-format-2.2.0 + source_url: https://github.com/apache/parquet-format/compare/parquet-format-2.1.0...apache-parquet-format-2.2.0 + source_display: 2.1.0..2.2.0 +"2.4.0": + tag: apache-parquet-format-2.4.0 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.3.1...apache-parquet-format-2.4.0 + source_display: 2.3.1..2.4.0 +"2.6.0": + tag: apache-parquet-format-2.6.0 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.5.0...apache-parquet-format-2.6.0 + source_display: 2.5.0..2.6.0 +"2.7.0": + tag: apache-parquet-format-2.7.0 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.6.0...apache-parquet-format-2.7.0 + source_display: 2.6.0..2.7.0 +"2.8.0": + tag: apache-parquet-format-2.8.0 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.7.0...apache-parquet-format-2.8.0 + source_display: 2.7.0..2.8.0 +"2.9.0": + tag: apache-parquet-format-2.9.0 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.8.0...apache-parquet-format-2.9.0 + source_display: 2.8.0..2.9.0 +"2.10.0": + tag: apache-parquet-format-2.10.0 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.9.0...apache-parquet-format-2.10.0 + source_display: 2.9.0..2.10.0 +"2.11.0": + tag: apache-parquet-format-2.11.0 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.10.0...apache-parquet-format-2.11.0 + source_display: 2.10.0..2.11.0 +"2.12.0": + tag: apache-parquet-format-2.12.0 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.11.0...apache-parquet-format-2.12.0 + source_display: 2.11.0..2.12.0 +"2.13.0": + tag: apache-parquet-format-2.13.0 + source_url: https://github.com/apache/parquet-format/compare/apache-parquet-format-2.12.0...apache-parquet-format-2.13.0 + source_display: 2.12.0..2.13.0 diff --git a/layouts/shortcodes/README.md b/layouts/shortcodes/README.md new file mode 100644 index 00000000..da45bd56 --- /dev/null +++ b/layouts/shortcodes/README.md @@ -0,0 +1,15 @@ +# Hugo shortcodes + +Shortcodes are Hugo templates called from Markdown content; +see the [Hugo shortcode documentation] for details. + +This site uses shortcodes to dynamically generate content from data stored in +the `data/` directory. + +For example the [format-versions.html] shortcode is used in the [versions.md] +page to generate the format version tables. The page calls it with Hugo's +shortcode syntax and passes a `table` parameter such as `forward_incompatible`. + +[format-versions.html]: format-versions.html +[versions.md]: ../../content/en/docs/File%20Format/versions.md +[Hugo shortcode documentation]: https://gohugo.io/content-management/shortcodes/#custom diff --git a/layouts/shortcodes/format-versions.html b/layouts/shortcodes/format-versions.html new file mode 100644 index 00000000..21e30f6e --- /dev/null +++ b/layouts/shortcodes/format-versions.html @@ -0,0 +1,95 @@ +{{- /* + Render Parquet format version feature tables from data/implementations/features/*.yaml. + See layouts/shortcodes/README.md for details. + Usage: {{< format-versions table="forward_incompatible" >}} +*/ -}} +{{- $tableName := .Get "table" -}} +{{- $validTables := slice "forward_incompatible" "forward_compatible" -}} +{{- if not (in $validTables $tableName) -}} + {{- errorf "unknown format versions table %q" $tableName -}} +{{- end -}} + +{{- /* Shared release-tag metadata keyed by feature released_in version. */ -}} +{{- $formatReleases := index site.Data.implementations "parquet-format-releases" -}} +{{- /* + Collect features from each category that belong in the requested compatibility table. + Hugo sorts strings lexicographically, so build a zero-padded numeric sort key from + released_in before sorting. For example, "0002.0010.0000" sorts after + "0002.0009.0000", while plain string sorting would place "2.10.0" before "2.9.0". + Append the feature id to make ordering stable when multiple features share a release. +*/ -}} +{{- $features := slice -}} +{{- range site.Data.implementations.categories -}} + {{- $featuresData := index site.Data.implementations.features .id -}} + {{- if $featuresData -}} + {{- range $featuresData.features -}} + {{- if eq .compatibility $tableName -}} + {{- $releasedIn := printf "%v" .released_in -}} + {{- $versionParts := split $releasedIn "." -}} + {{- $major := int (index $versionParts 0) -}} + {{- $minor := 0 -}} + {{- $patch := 0 -}} + {{- if ge (len $versionParts) 2 -}} + {{- $minor = int (index $versionParts 1) -}} + {{- end -}} + {{- if ge (len $versionParts) 3 -}} + {{- $patch = int (index $versionParts 2) -}} + {{- end -}} + {{- $sortKey := printf "%04d.%04d.%04d.%s" $major $minor $patch .id -}} + {{- $features = $features | append (dict "sort_key" $sortKey "feature" .) -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- $features = sort $features "sort_key" "desc" -}} + + + + + + + + + + + + + {{- range $features }} + {{- $feature := .feature -}} + {{- $displayName := or $feature.version_display_name $feature.display_name -}} + {{- $featureUrl := or $feature.feature_url $feature.spec_url -}} + {{- $releasedIn := printf "%v" $feature.released_in -}} + {{- /* Fail the build if a dated feature points at an unknown parquet-format release. */ -}} + {{- $release := index $formatReleases $releasedIn -}} + {{- if not $release -}} + {{- errorf "missing parquet-format release metadata for %q" $releasedIn -}} + {{- end -}} + {{- $releaseUrl := printf "https://github.com/apache/parquet-format/releases/tag/%s" $release.tag -}} + + + + + + + + {{- end }} + +
FeatureReleased inSourceApprovedNotes
+ {{- if $featureUrl -}} + {{ $displayName | safeHTML }} + {{- else -}} + {{ $displayName | safeHTML }} + {{- end -}} + + {{ $releasedIn }} + + {{ $release.source_display }} + + {{- with $feature.approval -}} + {{ . | markdownify }} + {{- end -}} + + {{- with $feature.version_note -}} + {{ . | markdownify }} + {{- end -}} +
diff --git a/layouts/shortcodes/implementation-status.html b/layouts/shortcodes/implementation-status.html index 0008c443..b2c9742b 100644 --- a/layouts/shortcodes/implementation-status.html +++ b/layouts/shortcodes/implementation-status.html @@ -1,11 +1,14 @@ {{- /* - Render implementation status tables from structured data in `data/implementation` + Render implementation status tables from structured data in `data/implementations` + See layouts/shortcodes/README.md for details. Usage: {{< implementation-status >}} */ -}} {{- $engines := site.Data.implementations.engines -}} {{- $categories := site.Data.implementations.categories -}} {{- $legend := site.Data.implementations.legend -}} +{{- /* Shared release-tag metadata keyed by feature released_in version. */ -}} +{{- $formatReleases := index site.Data.implementations "parquet-format-releases" -}} {{- /* Render each category */ -}} @@ -45,6 +48,8 @@

{{ $category.name }} {{- range $features -}} + {{- /* Include rows by default; skip rows explicitly marked implementation_status: false. */ -}} + {{- if ne .implementation_status false -}} {{- $feature := . -}} {{- $featureId := .id -}} @@ -56,10 +61,10 @@

{{ $category.name }} {{ .display_name }} {{- end -}} - {{- /* Render format version inline if present */ -}} - {{- if .format_version -}} + {{- /* Render format release year inline if present. */ -}} + {{- if .release_date -}} {{- $year := "" -}} - {{- $dateStr := .format_version.date -}} + {{- $dateStr := .release_date -}} {{- if hasPrefix $dateStr "2025" -}} {{- $year = "2025" -}} {{- else if hasPrefix $dateStr "2024" -}} @@ -75,7 +80,13 @@

{{ $category.name }} {{- else -}} {{- $year = substr $dateStr 0 4 -}} {{- end -}} - {{- $releaseUrl := printf "https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-%s" .format_version.version -}} + {{- $releasedIn := printf "%v" .released_in -}} + {{- /* Fail the build if a dated feature points at an unknown parquet-format release. */ -}} + {{- $release := index $formatReleases $releasedIn -}} + {{- if not $release -}} + {{- errorf "missing parquet-format release metadata for %q" $releasedIn -}} + {{- end -}} + {{- $releaseUrl := printf "https://github.com/apache/parquet-format/releases/tag/%s" $release.tag -}} ({{ $year }}) {{- end -}} @@ -236,6 +247,7 @@

{{ $category.name }} {{- end -}} + {{- end -}} {{- end -}} @@ -255,7 +267,7 @@

Minimum Version fo

Note: This data was originally collected in December 2025, and not all data was backfilled. It is likely older releases of each engine support reading all features for 2023 and before. As volunteers have time they are invited to add more granular details on releases. Generally, versions are expected to be accurate for any year 2025 and after.

-{{- /* Collect all features with format_version and group by year */ -}} +{{- /* Collect all features with release_date and group by year */ -}} {{- $featuresByYear := dict "2023" (slice) "2024" (slice) "2025" (slice) -}} {{- $excludedFeatures := slice "encoding-bit-packed" "compression-lz4-deprecated" "compression-lzo" "logical-enum" "logical-uuid" "logical-bson" "logical-json" "logical-interval" -}} {{- $includedCategories := slice "compressions" "encodings" "physical-types" "logical-types" -}} @@ -267,12 +279,14 @@

Minimum Version fo {{- $featuresData := index site.Data.implementations.features $categoryId -}} {{- if $featuresData -}} {{- range $featuresData.features -}} + {{- /* Include rows by default; skip rows explicitly marked implementation_status: false. */ -}} + {{- if ne .implementation_status false -}} {{- $feature := . -}} {{- /* Skip excluded features */ -}} {{- if not (in $excludedFeatures .id) -}} {{- $year := "" -}} - {{- if .format_version -}} - {{- $dateStr := .format_version.date -}} + {{- if .release_date -}} + {{- $dateStr := .release_date -}} {{- if hasPrefix $dateStr "2025" -}} {{- $year = "2025" -}} {{- else if hasPrefix $dateStr "2024" -}} @@ -281,13 +295,14 @@

Minimum Version fo {{- $year = "2023" -}} {{- end -}} {{- else -}} - {{- /* Features without format_version are assumed pre-2023 */ -}} + {{- /* Features without release_date are assumed pre-2023 */ -}} {{- $year = "2023" -}} {{- end -}} {{- $currentList := index $featuresByYear $year -}} {{- $featuresByYear = merge $featuresByYear (dict $year ($currentList | append $feature)) -}} {{- end -}} + {{- end -}} {{- end -}} {{- end -}} {{- end -}} @@ -301,9 +316,12 @@

Minimum Version fo {{- $featuresData := index site.Data.implementations.features $categoryId -}} {{- if $featuresData -}} {{- range $featuresData.features -}} + {{- /* Include rows by default; skip rows explicitly marked implementation_status: false. */ -}} + {{- if ne .implementation_status false -}} {{- if in $excludedFeatures .id -}} {{- $excludedFeatureNames = $excludedFeatureNames | append .display_name -}} {{- end -}} + {{- end -}} {{- end -}} {{- end -}} {{- end -}} @@ -353,6 +371,8 @@

Minimum Version fo {{- /* Find minimum version that supports all features in this year */ -}} {{- if $supportData -}} {{- range $features -}} + {{- /* Include rows by default; skip rows explicitly marked implementation_status: false. */ -}} + {{- if ne .implementation_status false -}} {{- $featureId := .id -}} {{- $support := index $supportData.support $featureId -}} @@ -398,6 +418,7 @@

Minimum Version fo {{- /* Feature is not supported or only has write support */ -}} {{- $missingFeatures = $missingFeatures | append .display_name -}} {{- end -}} + {{- end -}} {{- end -}} {{- /* Find release notes URL for minimum version */ -}}