From 669630cf056080c3360b1c2f7d802392ce8898fc Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 7 Jul 2026 09:38:32 -0400 Subject: [PATCH 1/5] Add PRs column linking features to their spec pull requests Adds a 'prs' field to each feature in the format version tables with links to the specific apache/parquet-format pull requests (or commits, for changes predating the use of pull requests) that added the feature to the specification. Closes #191 Co-Authored-By: Claude Fable 5 --- content/en/docs/File Format/versions.md | 5 +++- .../features/compressions.yaml | 8 ++++++ data/implementations/features/encodings.yaml | 10 ++++++++ .../features/format-features.yaml | 11 ++++++++ .../features/logical-types.yaml | 25 +++++++++++++++++++ .../features/physical-types.yaml | 8 ++++++ layouts/shortcodes/format-versions.html | 6 +++++ 7 files changed, 72 insertions(+), 1 deletion(-) diff --git a/content/en/docs/File Format/versions.md b/content/en/docs/File Format/versions.md index 00b7f0df..bdb3704b 100644 --- a/content/en/docs/File Format/versions.md +++ b/content/en/docs/File Format/versions.md @@ -72,7 +72,10 @@ the next parquet-format release. ## Forward incompatible features by version -Forward incompatible features and the format version each became available in: +Forward incompatible features and the format version each became available in. +The *PRs* column links to the specific pull requests that added each feature to +the specification. Features marked with a `*` were added before Parquet moved +to Apache in 2014, and the original pull request records no longer exist. {{< format-versions table="forward_incompatible" >}} diff --git a/data/implementations/features/compressions.yaml b/data/implementations/features/compressions.yaml index 8165683b..d8ff8f15 100644 --- a/data/implementations/features/compressions.yaml +++ b/data/implementations/features/compressions.yaml @@ -5,38 +5,46 @@ features: compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#uncompressed + prs: '\*' - id: compression-brotli display_name: BROTLI compatibility: forward_incompatible released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#brotli + prs: "[#40](https://github.com/apache/parquet-format/pull/40)" - id: compression-gzip display_name: GZIP compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#gzip + prs: '\*' - id: compression-lz4-deprecated display_name: LZ4 (deprecated) compatibility: forward_incompatible released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lz4 + prs: "[#70](https://github.com/apache/parquet-format/pull/70), [#168](https://github.com/apache/parquet-format/pull/168)" - id: compression-lz4-raw display_name: LZ4_RAW compatibility: forward_incompatible released_in: "2.9.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lz4_raw + prs: "[#168](https://github.com/apache/parquet-format/pull/168)" - id: compression-lzo display_name: LZO compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lzo + prs: '\*' - id: compression-snappy display_name: SNAPPY compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#snappy + prs: '\*' - id: compression-zstd display_name: ZSTD compatibility: forward_incompatible released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#zstd + prs: "[#70](https://github.com/apache/parquet-format/pull/70)" diff --git a/data/implementations/features/encodings.yaml b/data/implementations/features/encodings.yaml index e03eed38..b2007554 100644 --- a/data/implementations/features/encodings.yaml +++ b/data/implementations/features/encodings.yaml @@ -5,47 +5,56 @@ features: compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#plain-plain--0 + prs: '\*' - id: encoding-plain-dictionary display_name: PLAIN_DICTIONARY compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 + prs: '\*' - id: encoding-rle-dictionary display_name: RLE_DICTIONARY compatibility: forward_incompatible released_in: "2.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 + prs: '\*' - id: encoding-rle display_name: RLE compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3 + prs: '\*' - id: encoding-bit-packed display_name: BIT_PACKED (deprecated) compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#bit-packed-deprecated-bit_packed--4 + prs: '\*' - id: encoding-delta-binary-packed display_name: DELTA_BINARY_PACKED compatibility: forward_incompatible released_in: "2.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-encoding-delta_binary_packed--5 + prs: '\*' - id: encoding-delta-length-byte-array display_name: DELTA_LENGTH_BYTE_ARRAY compatibility: forward_incompatible released_in: "2.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-length-byte-array-delta_length_byte_array--6 + prs: '\*' - id: encoding-delta-byte-array display_name: DELTA_BYTE_ARRAY compatibility: forward_incompatible released_in: "2.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-strings-delta_byte_array--7 + prs: '\*' - id: encoding-byte-stream-split display_name: BYTE_STREAM_SPLIT compatibility: forward_incompatible released_in: "2.8.0" release_date: '2020-01-13' feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 + prs: "[#144](https://github.com/apache/parquet-format/pull/144)" approval: "[2019-12-03](https://lists.apache.org/thread/xs5qt2odm299pxgqb22mty2csc1so5yr)" - id: encoding-byte-stream-split-extended display_name: BYTE_STREAM_SPLIT (Additional Types) @@ -55,4 +64,5 @@ features: released_in: "2.11.0" release_date: '2024-03-21' feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 + prs: "[#229](https://github.com/apache/parquet-format/pull/229)" approval: "[2024-03-18](https://lists.apache.org/thread/nlsj0ftxy7y4ov1678rgy5zc7dmogg6q)" diff --git a/data/implementations/features/format-features.yaml b/data/implementations/features/format-features.yaml index b45a5847..22447781 100644 --- a/data/implementations/features/format-features.yaml +++ b/data/implementations/features/format-features.yaml @@ -7,6 +7,7 @@ features: released_in: "2.7.0" release_date: '2019-09-25' feature_url: https://github.com/apache/parquet-format/blob/master/BloomFilter.md + prs: "[#112](https://github.com/apache/parquet-format/pull/112), [#145](https://github.com/apache/parquet-format/pull/145)" approval: "[2019-09-09](https://lists.apache.org/thread/ktdx1xp0d2gjfgkcvd29zxvt3cgg88bo)" - id: format-bloom-filter-length display_name: Bloom filter length @@ -14,6 +15,7 @@ features: compatibility: forward_compatible released_in: "2.10.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L933 + prs: "[#194](https://github.com/apache/parquet-format/pull/194)" - id: format-stats-min-max display_name: Statistics min_value, max_value - id: format-page-index @@ -23,11 +25,13 @@ features: released_in: "2.4.0" release_date: '2018-05-04' feature_url: https://github.com/apache/parquet-format/blob/master/PageIndex.md + prs: "[#72](https://github.com/apache/parquet-format/pull/72)" - id: format-page-crc32 display_name: Page CRC32 checksum compatibility: forward_compatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L829 + prs: '\*' - id: format-modular-encryption display_name: Modular encryption spec_url: https://github.com/apache/parquet-format/blob/master/Encryption.md @@ -35,6 +39,7 @@ features: released_in: "2.7.0" release_date: '2019-09-25' feature_url: https://github.com/apache/parquet-format/blob/master/Encryption.md + prs: "[#94](https://github.com/apache/parquet-format/pull/94), [#142](https://github.com/apache/parquet-format/pull/142)" approval: "[2019-01-16](https://lists.apache.org/thread/l8zcwnbrnhjh3w2k1lyb0v6ct5lnzr0h)" - id: format-size-statistics display_name: Size statistics @@ -43,6 +48,7 @@ features: released_in: "2.10.0" release_date: '2023-11-20' feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L202 + prs: "[#197](https://github.com/apache/parquet-format/pull/197)" approval: "[2023-11-14](https://lists.apache.org/thread/wgobz41mfldbhqpg9q4mdwypghg2cxg2)" - id: format-data-page-v2 display_name: Data Page V2 @@ -50,18 +56,21 @@ features: compatibility: forward_incompatible released_in: "2.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L724 + prs: '\*' - id: format-data-page-v1 display_name: Data Page V1 implementation_status: false compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L671 + prs: '\*' - id: format-geospatial-statistics display_name: Geospatial statistics implementation_status: false compatibility: forward_compatible released_in: "2.11.0" feature_url: https://github.com/apache/parquet-format/blob/master/Geospatial.md#statistics + prs: "[#240](https://github.com/apache/parquet-format/pull/240)" approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: format-binary-protocol-extensions display_name: Binary protocol extensions @@ -69,6 +78,7 @@ features: compatibility: forward_compatible released_in: "2.11.0" feature_url: https://github.com/apache/parquet-format/blob/master/BinaryProtocolExtensions.md + prs: "[#254](https://github.com/apache/parquet-format/pull/254)" approval: "[2024-09-06](https://lists.apache.org/thread/x3472kldrq5kjnld9ztj1jozz25f40hg)" - id: format-ieee-754-total-order-and-nan-counts display_name: IEEE 754 total order and NaN counts @@ -76,4 +86,5 @@ features: compatibility: forward_compatible released_in: "2.13.0" feature_url: https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift#L1061 + prs: "[#514](https://github.com/apache/parquet-format/pull/514)" approval: "[2026-05-26](https://lists.apache.org/thread/h0k0hqo0sojqphnbnrkp8b0gmwdzq9on)" diff --git a/data/implementations/features/logical-types.yaml b/data/implementations/features/logical-types.yaml index 9adf65dc..394a4872 100644 --- a/data/implementations/features/logical-types.yaml +++ b/data/implementations/features/logical-types.yaml @@ -6,44 +6,52 @@ features: compatibility: forward_compatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#string + prs: '\*' - id: logical-enum display_name: ENUM version_display_name: ENUM (BYTE_ARRAY) compatibility: forward_compatible released_in: "2.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#enum + prs: '\*' - id: logical-uuid display_name: UUID version_display_name: UUID (FIXED_LEN_BYTE_ARRAY(16)) compatibility: forward_compatible released_in: "2.6.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#uuid + prs: "[#71](https://github.com/apache/parquet-format/pull/71), [#93](https://github.com/apache/parquet-format/pull/93)" - id: logical-int-types display_name: 8, 16, 32, 64 bit signed and unsigned INT version_display_name: Signed and unsigned integer logical types (INT32, INT64) compatibility: forward_compatible released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#signed-integers + prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5), [#51](https://github.com/apache/parquet-format/pull/51)" - id: logical-decimal-int32 display_name: DECIMAL (INT32) compatibility: forward_compatible released_in: "2.1.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal + prs: '\*' - id: logical-decimal-int64 display_name: DECIMAL (INT64) compatibility: forward_compatible released_in: "2.1.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal + prs: '\*' - id: logical-decimal-byte-array display_name: DECIMAL (BYTE_ARRAY) compatibility: forward_compatible released_in: "2.1.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal + prs: '\*' - id: logical-decimal-fixed-len-byte-array display_name: DECIMAL (FIXED_LEN_BYTE_ARRAY) compatibility: forward_compatible released_in: "2.1.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal + prs: '\*' - id: logical-float16 display_name: FLOAT16 version_display_name: FLOAT16 (FIXED_LEN_BYTE_ARRAY(2)) @@ -51,6 +59,7 @@ features: released_in: "2.10.0" release_date: '2023-11-20' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#float16 + prs: "[#184](https://github.com/apache/parquet-format/pull/184)" approval: "[2023-10-13](https://lists.apache.org/thread/gyvqcx9ssxkjlrwogqwy7n4z6ofdm871)" - id: logical-date display_name: DATE @@ -58,39 +67,46 @@ features: compatibility: forward_compatible released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#date + prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5)" - id: logical-time-int32 display_name: TIME (INT32) compatibility: forward_compatible released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time + prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5)" - id: logical-time-int64 display_name: TIME (INT64) compatibility: forward_compatible released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time + prs: "[#23](https://github.com/apache/parquet-format/pull/23)" - id: logical-timestamp-int64 display_name: TIMESTAMP (INT64) compatibility: forward_compatible released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#timestamp + prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5), [#23](https://github.com/apache/parquet-format/pull/23)" - id: logical-interval display_name: INTERVAL version_display_name: INTERVAL (FIXED_LEN_BYTE_ARRAY(12)) compatibility: forward_compatible released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#interval + prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5)" - id: logical-json display_name: JSON version_display_name: JSON (BYTE_ARRAY) compatibility: forward_compatible released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#json + prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5)" - id: logical-bson display_name: BSON version_display_name: BSON (BYTE_ARRAY) compatibility: forward_compatible released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#bson + prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5)" - id: logical-variant display_name: VARIANT spec_url: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md @@ -98,6 +114,7 @@ features: released_in: "2.12.0" release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md + prs: "[#456](https://github.com/apache/parquet-format/pull/456), [#460](https://github.com/apache/parquet-format/pull/460)" approval: "[2025-08-24](https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7)" - id: logical-geometry display_name: GEOMETRY @@ -107,6 +124,7 @@ features: released_in: "2.11.0" release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geometry + prs: "[#240](https://github.com/apache/parquet-format/pull/240)" approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: logical-geography display_name: GEOGRAPHY @@ -116,28 +134,33 @@ features: released_in: "2.11.0" release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geography + prs: "[#240](https://github.com/apache/parquet-format/pull/240)" approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: logical-list display_name: LIST compatibility: forward_compatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists + prs: "[#17](https://github.com/apache/parquet-format/pull/17)" - id: logical-map display_name: MAP compatibility: forward_compatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#maps + prs: "[#17](https://github.com/apache/parquet-format/pull/17)" - id: logical-unknown display_name: UNKNOWN (always null) compatibility: forward_compatible released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#unknown-always-null + prs: "[#51](https://github.com/apache/parquet-format/pull/51)" - id: logical-type-union display_name: LogicalType union implementation_status: false compatibility: forward_compatible released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L471 + prs: "[#51](https://github.com/apache/parquet-format/pull/51)" version_note: Supersedes `ConvertedType` enum
deprecated in [2.9.0](https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.9.0) - id: logical-nanosecond-time-timestamp display_name: Nanosecond TIME/TIMESTAMP @@ -145,10 +168,12 @@ features: compatibility: forward_compatible released_in: "2.6.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L352 + prs: "[#102](https://github.com/apache/parquet-format/pull/102)" - id: logical-variant-shredding display_name: Variant shredding implementation_status: false compatibility: forward_compatible released_in: "2.12.0" feature_url: https://github.com/apache/parquet-format/blob/master/VariantShredding.md + prs: "[#456](https://github.com/apache/parquet-format/pull/456), [#461](https://github.com/apache/parquet-format/pull/461)" approval: "[2025-08-24](https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7)" diff --git a/data/implementations/features/physical-types.yaml b/data/implementations/features/physical-types.yaml index 3c0b1bcb..d8d1347c 100644 --- a/data/implementations/features/physical-types.yaml +++ b/data/implementations/features/physical-types.yaml @@ -5,16 +5,19 @@ features: compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L33 + prs: '\*' - id: physical-int32 display_name: INT32 compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L34 + prs: '\*' - id: physical-int64 display_name: INT64 compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L35 + prs: '\*' - id: physical-int96 display_name: INT96 note: This type is deprecated, but as of 2024 it's common in currently produced parquet files @@ -22,23 +25,28 @@ features: compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L36 + prs: '\*' - id: physical-float display_name: FLOAT compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L37 + prs: '\*' - id: physical-double display_name: DOUBLE compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L38 + prs: '\*' - id: physical-byte-array display_name: BYTE_ARRAY compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L39 + prs: '\*' - id: physical-fixed-len-byte-array display_name: FIXED_LEN_BYTE_ARRAY compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L40 + prs: '\*' diff --git a/layouts/shortcodes/format-versions.html b/layouts/shortcodes/format-versions.html index 21e30f6e..fc77deae 100644 --- a/layouts/shortcodes/format-versions.html +++ b/layouts/shortcodes/format-versions.html @@ -49,6 +49,7 @@ Feature Released in Source + PRs Approved Notes @@ -79,6 +80,11 @@ {{ $release.source_display }} + + {{- with $feature.prs -}} + {{ . | markdownify }} + {{- end -}} + {{- with $feature.approval -}} {{ . | markdownify }} From b4aaf006b432e485f8c8a410b9aea91713ff456d Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 7 Jul 2026 11:56:21 -0400 Subject: [PATCH 2/5] Add approval threads for more features Adds mailing list approval/consensus threads found in the dev@parquet.apache.org archives for the date/time logical types, the LogicalType union, and LZ4_RAW. Co-Authored-By: Claude Fable 5 --- data/implementations/features/compressions.yaml | 1 + data/implementations/features/logical-types.yaml | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/data/implementations/features/compressions.yaml b/data/implementations/features/compressions.yaml index d8ff8f15..455bb6b7 100644 --- a/data/implementations/features/compressions.yaml +++ b/data/implementations/features/compressions.yaml @@ -30,6 +30,7 @@ features: released_in: "2.9.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lz4_raw prs: "[#168](https://github.com/apache/parquet-format/pull/168)" + approval: "[2021-02-16](https://lists.apache.org/thread/l15qq12v38w9jnkd6p9mdd11kr0nq3gr)" - id: compression-lzo display_name: LZO compatibility: forward_incompatible diff --git a/data/implementations/features/logical-types.yaml b/data/implementations/features/logical-types.yaml index 394a4872..06151f4d 100644 --- a/data/implementations/features/logical-types.yaml +++ b/data/implementations/features/logical-types.yaml @@ -68,12 +68,14 @@ features: released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#date prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5)" + approval: "[2014-07-10](https://lists.apache.org/thread/fpozjxyxws42d0fw7n27xobjjdtrqwqq)" - id: logical-time-int32 display_name: TIME (INT32) compatibility: forward_compatible released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5)" + approval: "[2014-07-10](https://lists.apache.org/thread/fpozjxyxws42d0fw7n27xobjjdtrqwqq)" - id: logical-time-int64 display_name: TIME (INT64) compatibility: forward_compatible @@ -86,6 +88,7 @@ features: released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#timestamp prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5), [#23](https://github.com/apache/parquet-format/pull/23)" + approval: "[2014-07-10](https://lists.apache.org/thread/fpozjxyxws42d0fw7n27xobjjdtrqwqq)" - id: logical-interval display_name: INTERVAL version_display_name: INTERVAL (FIXED_LEN_BYTE_ARRAY(12)) @@ -93,6 +96,7 @@ features: released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#interval prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5)" + approval: "[2014-07-10](https://lists.apache.org/thread/fpozjxyxws42d0fw7n27xobjjdtrqwqq)" - id: logical-json display_name: JSON version_display_name: JSON (BYTE_ARRAY) @@ -154,6 +158,7 @@ features: released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#unknown-always-null prs: "[#51](https://github.com/apache/parquet-format/pull/51)" + approval: "[2017-07-19](https://lists.apache.org/thread/j3yjr59jl94c71r7s778osm8mbcc8lcn)" - id: logical-type-union display_name: LogicalType union implementation_status: false @@ -161,6 +166,7 @@ features: released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L471 prs: "[#51](https://github.com/apache/parquet-format/pull/51)" + approval: "[2017-07-19](https://lists.apache.org/thread/j3yjr59jl94c71r7s778osm8mbcc8lcn)" version_note: Supersedes `ConvertedType` enum
deprecated in [2.9.0](https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.9.0) - id: logical-nanosecond-time-timestamp display_name: Nanosecond TIME/TIMESTAMP From 31bec1b2a8057b188f570f47d3e57defec8a1a89 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 7 Jul 2026 11:59:01 -0400 Subject: [PATCH 3/5] Leave PRs column blank for pre-Apache features Co-Authored-By: Claude Fable 5 --- content/en/docs/File Format/versions.md | 4 ++-- data/implementations/features/compressions.yaml | 4 ---- data/implementations/features/encodings.yaml | 8 -------- data/implementations/features/format-features.yaml | 3 --- data/implementations/features/logical-types.yaml | 6 ------ data/implementations/features/physical-types.yaml | 8 -------- 6 files changed, 2 insertions(+), 31 deletions(-) diff --git a/content/en/docs/File Format/versions.md b/content/en/docs/File Format/versions.md index bdb3704b..d5aa31db 100644 --- a/content/en/docs/File Format/versions.md +++ b/content/en/docs/File Format/versions.md @@ -74,8 +74,8 @@ the next parquet-format release. Forward incompatible features and the format version each became available in. The *PRs* column links to the specific pull requests that added each feature to -the specification. Features marked with a `*` were added before Parquet moved -to Apache in 2014, and the original pull request records no longer exist. +the specification. Features added before Parquet moved to Apache in 2014 have +no entry, as the original pull request records no longer exist. {{< format-versions table="forward_incompatible" >}} diff --git a/data/implementations/features/compressions.yaml b/data/implementations/features/compressions.yaml index 455bb6b7..e5000df0 100644 --- a/data/implementations/features/compressions.yaml +++ b/data/implementations/features/compressions.yaml @@ -5,7 +5,6 @@ features: compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#uncompressed - prs: '\*' - id: compression-brotli display_name: BROTLI compatibility: forward_incompatible @@ -17,7 +16,6 @@ features: compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#gzip - prs: '\*' - id: compression-lz4-deprecated display_name: LZ4 (deprecated) compatibility: forward_incompatible @@ -36,13 +34,11 @@ features: compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lzo - prs: '\*' - id: compression-snappy display_name: SNAPPY compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#snappy - prs: '\*' - id: compression-zstd display_name: ZSTD compatibility: forward_incompatible diff --git a/data/implementations/features/encodings.yaml b/data/implementations/features/encodings.yaml index b2007554..7db7fca9 100644 --- a/data/implementations/features/encodings.yaml +++ b/data/implementations/features/encodings.yaml @@ -5,49 +5,41 @@ features: compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#plain-plain--0 - prs: '\*' - id: encoding-plain-dictionary display_name: PLAIN_DICTIONARY compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 - prs: '\*' - id: encoding-rle-dictionary display_name: RLE_DICTIONARY compatibility: forward_incompatible released_in: "2.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8 - prs: '\*' - id: encoding-rle display_name: RLE compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#run-length-encoding--bit-packing-hybrid-rle--3 - prs: '\*' - id: encoding-bit-packed display_name: BIT_PACKED (deprecated) compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#bit-packed-deprecated-bit_packed--4 - prs: '\*' - id: encoding-delta-binary-packed display_name: DELTA_BINARY_PACKED compatibility: forward_incompatible released_in: "2.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-encoding-delta_binary_packed--5 - prs: '\*' - id: encoding-delta-length-byte-array display_name: DELTA_LENGTH_BYTE_ARRAY compatibility: forward_incompatible released_in: "2.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-length-byte-array-delta_length_byte_array--6 - prs: '\*' - id: encoding-delta-byte-array display_name: DELTA_BYTE_ARRAY compatibility: forward_incompatible released_in: "2.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#delta-strings-delta_byte_array--7 - prs: '\*' - id: encoding-byte-stream-split display_name: BYTE_STREAM_SPLIT compatibility: forward_incompatible diff --git a/data/implementations/features/format-features.yaml b/data/implementations/features/format-features.yaml index 22447781..0fb081ad 100644 --- a/data/implementations/features/format-features.yaml +++ b/data/implementations/features/format-features.yaml @@ -31,7 +31,6 @@ features: compatibility: forward_compatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L829 - prs: '\*' - id: format-modular-encryption display_name: Modular encryption spec_url: https://github.com/apache/parquet-format/blob/master/Encryption.md @@ -56,14 +55,12 @@ features: compatibility: forward_incompatible released_in: "2.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L724 - prs: '\*' - id: format-data-page-v1 display_name: Data Page V1 implementation_status: false compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L671 - prs: '\*' - id: format-geospatial-statistics display_name: Geospatial statistics implementation_status: false diff --git a/data/implementations/features/logical-types.yaml b/data/implementations/features/logical-types.yaml index 06151f4d..ada10a04 100644 --- a/data/implementations/features/logical-types.yaml +++ b/data/implementations/features/logical-types.yaml @@ -6,14 +6,12 @@ features: compatibility: forward_compatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#string - prs: '\*' - id: logical-enum display_name: ENUM version_display_name: ENUM (BYTE_ARRAY) compatibility: forward_compatible released_in: "2.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#enum - prs: '\*' - id: logical-uuid display_name: UUID version_display_name: UUID (FIXED_LEN_BYTE_ARRAY(16)) @@ -33,25 +31,21 @@ features: compatibility: forward_compatible released_in: "2.1.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal - prs: '\*' - id: logical-decimal-int64 display_name: DECIMAL (INT64) compatibility: forward_compatible released_in: "2.1.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal - prs: '\*' - id: logical-decimal-byte-array display_name: DECIMAL (BYTE_ARRAY) compatibility: forward_compatible released_in: "2.1.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal - prs: '\*' - id: logical-decimal-fixed-len-byte-array display_name: DECIMAL (FIXED_LEN_BYTE_ARRAY) compatibility: forward_compatible released_in: "2.1.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#decimal - prs: '\*' - id: logical-float16 display_name: FLOAT16 version_display_name: FLOAT16 (FIXED_LEN_BYTE_ARRAY(2)) diff --git a/data/implementations/features/physical-types.yaml b/data/implementations/features/physical-types.yaml index d8d1347c..3c0b1bcb 100644 --- a/data/implementations/features/physical-types.yaml +++ b/data/implementations/features/physical-types.yaml @@ -5,19 +5,16 @@ features: compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L33 - prs: '\*' - id: physical-int32 display_name: INT32 compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L34 - prs: '\*' - id: physical-int64 display_name: INT64 compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L35 - prs: '\*' - id: physical-int96 display_name: INT96 note: This type is deprecated, but as of 2024 it's common in currently produced parquet files @@ -25,28 +22,23 @@ features: compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L36 - prs: '\*' - id: physical-float display_name: FLOAT compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L37 - prs: '\*' - id: physical-double display_name: DOUBLE compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L38 - prs: '\*' - id: physical-byte-array display_name: BYTE_ARRAY compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L39 - prs: '\*' - id: physical-fixed-len-byte-array display_name: FIXED_LEN_BYTE_ARRAY compatibility: forward_incompatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L40 - prs: '\*' From 84c1631fff760e57974a18ba31d7a53281c08d29 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 7 Jul 2026 12:13:39 -0400 Subject: [PATCH 4/5] adjust wording --- content/en/docs/File Format/versions.md | 13 +++++++++---- layouts/shortcodes/format-versions.html | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/content/en/docs/File Format/versions.md b/content/en/docs/File Format/versions.md index d5aa31db..138a2bad 100644 --- a/content/en/docs/File Format/versions.md +++ b/content/en/docs/File Format/versions.md @@ -72,10 +72,15 @@ the next parquet-format release. ## Forward incompatible features by version -Forward incompatible features and the format version each became available in. -The *PRs* column links to the specific pull requests that added each feature to -the specification. Features added before Parquet moved to Apache in 2014 have -no entry, as the original pull request records no longer exist. +Forward incompatible features and the format version each became available in. +Columns include: + +* **Release**: The parquet-format version the feature was released in. +* **Source**: Link to all changes in the parquet-format release. +* **PRs**: The specific pull requests that added the feature to the + specification (features added before Parquet moved to Apache in 2014 have no + entry, as the original pull request records no longer exist). +* **Approved**: The mailing list thread where the feature was approved. {{< format-versions table="forward_incompatible" >}} diff --git a/layouts/shortcodes/format-versions.html b/layouts/shortcodes/format-versions.html index fc77deae..95612452 100644 --- a/layouts/shortcodes/format-versions.html +++ b/layouts/shortcodes/format-versions.html @@ -47,7 +47,7 @@ Feature - Released in + Release Source PRs Approved From 3744f776ad53d2bbd841a0c9c42f013e1e34fc31 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 7 Jul 2026 12:21:45 -0400 Subject: [PATCH 5/5] Add column definitions, rename Release column, prune non-vote links Co-Authored-By: Claude Fable 5 --- data/implementations/features/compressions.yaml | 1 - data/implementations/features/logical-types.yaml | 8 -------- 2 files changed, 9 deletions(-) diff --git a/data/implementations/features/compressions.yaml b/data/implementations/features/compressions.yaml index e5000df0..df4b117c 100644 --- a/data/implementations/features/compressions.yaml +++ b/data/implementations/features/compressions.yaml @@ -28,7 +28,6 @@ features: released_in: "2.9.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lz4_raw prs: "[#168](https://github.com/apache/parquet-format/pull/168)" - approval: "[2021-02-16](https://lists.apache.org/thread/l15qq12v38w9jnkd6p9mdd11kr0nq3gr)" - id: compression-lzo display_name: LZO compatibility: forward_incompatible diff --git a/data/implementations/features/logical-types.yaml b/data/implementations/features/logical-types.yaml index ada10a04..b788e3e4 100644 --- a/data/implementations/features/logical-types.yaml +++ b/data/implementations/features/logical-types.yaml @@ -62,14 +62,12 @@ features: released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#date prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5)" - approval: "[2014-07-10](https://lists.apache.org/thread/fpozjxyxws42d0fw7n27xobjjdtrqwqq)" - id: logical-time-int32 display_name: TIME (INT32) compatibility: forward_compatible released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5)" - approval: "[2014-07-10](https://lists.apache.org/thread/fpozjxyxws42d0fw7n27xobjjdtrqwqq)" - id: logical-time-int64 display_name: TIME (INT64) compatibility: forward_compatible @@ -82,7 +80,6 @@ features: released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#timestamp prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5), [#23](https://github.com/apache/parquet-format/pull/23)" - approval: "[2014-07-10](https://lists.apache.org/thread/fpozjxyxws42d0fw7n27xobjjdtrqwqq)" - id: logical-interval display_name: INTERVAL version_display_name: INTERVAL (FIXED_LEN_BYTE_ARRAY(12)) @@ -90,7 +87,6 @@ features: released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#interval prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5)" - approval: "[2014-07-10](https://lists.apache.org/thread/fpozjxyxws42d0fw7n27xobjjdtrqwqq)" - id: logical-json display_name: JSON version_display_name: JSON (BYTE_ARRAY) @@ -139,20 +135,17 @@ features: compatibility: forward_compatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists - prs: "[#17](https://github.com/apache/parquet-format/pull/17)" - id: logical-map display_name: MAP compatibility: forward_compatible released_in: "1.0.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#maps - prs: "[#17](https://github.com/apache/parquet-format/pull/17)" - id: logical-unknown display_name: UNKNOWN (always null) compatibility: forward_compatible released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#unknown-always-null prs: "[#51](https://github.com/apache/parquet-format/pull/51)" - approval: "[2017-07-19](https://lists.apache.org/thread/j3yjr59jl94c71r7s778osm8mbcc8lcn)" - id: logical-type-union display_name: LogicalType union implementation_status: false @@ -160,7 +153,6 @@ features: released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L471 prs: "[#51](https://github.com/apache/parquet-format/pull/51)" - approval: "[2017-07-19](https://lists.apache.org/thread/j3yjr59jl94c71r7s778osm8mbcc8lcn)" version_note: Supersedes `ConvertedType` enum
deprecated in [2.9.0](https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.9.0) - id: logical-nanosecond-time-timestamp display_name: Nanosecond TIME/TIMESTAMP