diff --git a/content/en/docs/File Format/versions.md b/content/en/docs/File Format/versions.md index 00b7f0df..138a2bad 100644 --- a/content/en/docs/File Format/versions.md +++ b/content/en/docs/File Format/versions.md @@ -72,7 +72,15 @@ the next parquet-format release. ## Forward incompatible features by version -Forward incompatible features and the format version each became available in: +Forward incompatible features and the format version each became available in. +Columns include: + +* **Release**: The parquet-format version the feature was released in. +* **Source**: Link to all changes in the parquet-format release. +* **PRs**: The specific pull requests that added the feature to the + specification (features added before Parquet moved to Apache in 2014 have no + entry, as the original pull request records no longer exist). +* **Approved**: The mailing list thread where the feature was approved. {{< format-versions table="forward_incompatible" >}} diff --git a/data/implementations/features/compressions.yaml b/data/implementations/features/compressions.yaml index 8165683b..df4b117c 100644 --- a/data/implementations/features/compressions.yaml +++ b/data/implementations/features/compressions.yaml @@ -10,6 +10,7 @@ features: compatibility: forward_incompatible released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#brotli + prs: "[#40](https://github.com/apache/parquet-format/pull/40)" - id: compression-gzip display_name: GZIP compatibility: forward_incompatible @@ -20,11 +21,13 @@ features: compatibility: forward_incompatible released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lz4 + prs: "[#70](https://github.com/apache/parquet-format/pull/70), [#168](https://github.com/apache/parquet-format/pull/168)" - id: compression-lz4-raw display_name: LZ4_RAW compatibility: forward_incompatible released_in: "2.9.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#lz4_raw + prs: "[#168](https://github.com/apache/parquet-format/pull/168)" - id: compression-lzo display_name: LZO compatibility: forward_incompatible @@ -40,3 +43,4 @@ features: compatibility: forward_incompatible released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/master/Compression.md#zstd + prs: "[#70](https://github.com/apache/parquet-format/pull/70)" diff --git a/data/implementations/features/encodings.yaml b/data/implementations/features/encodings.yaml index e03eed38..7db7fca9 100644 --- a/data/implementations/features/encodings.yaml +++ b/data/implementations/features/encodings.yaml @@ -46,6 +46,7 @@ features: released_in: "2.8.0" release_date: '2020-01-13' feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 + prs: "[#144](https://github.com/apache/parquet-format/pull/144)" approval: "[2019-12-03](https://lists.apache.org/thread/xs5qt2odm299pxgqb22mty2csc1so5yr)" - id: encoding-byte-stream-split-extended display_name: BYTE_STREAM_SPLIT (Additional Types) @@ -55,4 +56,5 @@ features: released_in: "2.11.0" release_date: '2024-03-21' feature_url: https://github.com/apache/parquet-format/blob/master/Encodings.md#byte-stream-split-byte_stream_split--9 + prs: "[#229](https://github.com/apache/parquet-format/pull/229)" approval: "[2024-03-18](https://lists.apache.org/thread/nlsj0ftxy7y4ov1678rgy5zc7dmogg6q)" diff --git a/data/implementations/features/format-features.yaml b/data/implementations/features/format-features.yaml index b45a5847..0fb081ad 100644 --- a/data/implementations/features/format-features.yaml +++ b/data/implementations/features/format-features.yaml @@ -7,6 +7,7 @@ features: released_in: "2.7.0" release_date: '2019-09-25' feature_url: https://github.com/apache/parquet-format/blob/master/BloomFilter.md + prs: "[#112](https://github.com/apache/parquet-format/pull/112), [#145](https://github.com/apache/parquet-format/pull/145)" approval: "[2019-09-09](https://lists.apache.org/thread/ktdx1xp0d2gjfgkcvd29zxvt3cgg88bo)" - id: format-bloom-filter-length display_name: Bloom filter length @@ -14,6 +15,7 @@ features: compatibility: forward_compatible released_in: "2.10.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L933 + prs: "[#194](https://github.com/apache/parquet-format/pull/194)" - id: format-stats-min-max display_name: Statistics min_value, max_value - id: format-page-index @@ -23,6 +25,7 @@ features: released_in: "2.4.0" release_date: '2018-05-04' feature_url: https://github.com/apache/parquet-format/blob/master/PageIndex.md + prs: "[#72](https://github.com/apache/parquet-format/pull/72)" - id: format-page-crc32 display_name: Page CRC32 checksum compatibility: forward_compatible @@ -35,6 +38,7 @@ features: released_in: "2.7.0" release_date: '2019-09-25' feature_url: https://github.com/apache/parquet-format/blob/master/Encryption.md + prs: "[#94](https://github.com/apache/parquet-format/pull/94), [#142](https://github.com/apache/parquet-format/pull/142)" approval: "[2019-01-16](https://lists.apache.org/thread/l8zcwnbrnhjh3w2k1lyb0v6ct5lnzr0h)" - id: format-size-statistics display_name: Size statistics @@ -43,6 +47,7 @@ features: released_in: "2.10.0" release_date: '2023-11-20' feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L202 + prs: "[#197](https://github.com/apache/parquet-format/pull/197)" approval: "[2023-11-14](https://lists.apache.org/thread/wgobz41mfldbhqpg9q4mdwypghg2cxg2)" - id: format-data-page-v2 display_name: Data Page V2 @@ -62,6 +67,7 @@ features: compatibility: forward_compatible released_in: "2.11.0" feature_url: https://github.com/apache/parquet-format/blob/master/Geospatial.md#statistics + prs: "[#240](https://github.com/apache/parquet-format/pull/240)" approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: format-binary-protocol-extensions display_name: Binary protocol extensions @@ -69,6 +75,7 @@ features: compatibility: forward_compatible released_in: "2.11.0" feature_url: https://github.com/apache/parquet-format/blob/master/BinaryProtocolExtensions.md + prs: "[#254](https://github.com/apache/parquet-format/pull/254)" approval: "[2024-09-06](https://lists.apache.org/thread/x3472kldrq5kjnld9ztj1jozz25f40hg)" - id: format-ieee-754-total-order-and-nan-counts display_name: IEEE 754 total order and NaN counts @@ -76,4 +83,5 @@ features: compatibility: forward_compatible released_in: "2.13.0" feature_url: https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift#L1061 + prs: "[#514](https://github.com/apache/parquet-format/pull/514)" approval: "[2026-05-26](https://lists.apache.org/thread/h0k0hqo0sojqphnbnrkp8b0gmwdzq9on)" diff --git a/data/implementations/features/logical-types.yaml b/data/implementations/features/logical-types.yaml index 9adf65dc..b788e3e4 100644 --- a/data/implementations/features/logical-types.yaml +++ b/data/implementations/features/logical-types.yaml @@ -18,12 +18,14 @@ features: compatibility: forward_compatible released_in: "2.6.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#uuid + prs: "[#71](https://github.com/apache/parquet-format/pull/71), [#93](https://github.com/apache/parquet-format/pull/93)" - id: logical-int-types display_name: 8, 16, 32, 64 bit signed and unsigned INT version_display_name: Signed and unsigned integer logical types (INT32, INT64) compatibility: forward_compatible released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#signed-integers + prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5), [#51](https://github.com/apache/parquet-format/pull/51)" - id: logical-decimal-int32 display_name: DECIMAL (INT32) compatibility: forward_compatible @@ -51,6 +53,7 @@ features: released_in: "2.10.0" release_date: '2023-11-20' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#float16 + prs: "[#184](https://github.com/apache/parquet-format/pull/184)" approval: "[2023-10-13](https://lists.apache.org/thread/gyvqcx9ssxkjlrwogqwy7n4z6ofdm871)" - id: logical-date display_name: DATE @@ -58,39 +61,46 @@ features: compatibility: forward_compatible released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#date + prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5)" - id: logical-time-int32 display_name: TIME (INT32) compatibility: forward_compatible released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time + prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5)" - id: logical-time-int64 display_name: TIME (INT64) compatibility: forward_compatible released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#time + prs: "[#23](https://github.com/apache/parquet-format/pull/23)" - id: logical-timestamp-int64 display_name: TIMESTAMP (INT64) compatibility: forward_compatible released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#timestamp + prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5), [#23](https://github.com/apache/parquet-format/pull/23)" - id: logical-interval display_name: INTERVAL version_display_name: INTERVAL (FIXED_LEN_BYTE_ARRAY(12)) compatibility: forward_compatible released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#interval + prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5)" - id: logical-json display_name: JSON version_display_name: JSON (BYTE_ARRAY) compatibility: forward_compatible released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#json + prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5)" - id: logical-bson display_name: BSON version_display_name: BSON (BYTE_ARRAY) compatibility: forward_compatible released_in: "2.2.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#bson + prs: "[#3](https://github.com/apache/parquet-format/pull/3), [#5](https://github.com/apache/parquet-format/pull/5)" - id: logical-variant display_name: VARIANT spec_url: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md @@ -98,6 +108,7 @@ features: released_in: "2.12.0" release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md + prs: "[#456](https://github.com/apache/parquet-format/pull/456), [#460](https://github.com/apache/parquet-format/pull/460)" approval: "[2025-08-24](https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7)" - id: logical-geometry display_name: GEOMETRY @@ -107,6 +118,7 @@ features: released_in: "2.11.0" release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geometry + prs: "[#240](https://github.com/apache/parquet-format/pull/240)" approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: logical-geography display_name: GEOGRAPHY @@ -116,6 +128,7 @@ features: released_in: "2.11.0" release_date: '2025-03-23' feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#geography + prs: "[#240](https://github.com/apache/parquet-format/pull/240)" approval: "[2025-02-09](https://lists.apache.org/thread/s6s714c98cn9gg22mnk5nsn7xymym8xo)" - id: logical-list display_name: LIST @@ -132,12 +145,14 @@ features: compatibility: forward_compatible released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#unknown-always-null + prs: "[#51](https://github.com/apache/parquet-format/pull/51)" - id: logical-type-union display_name: LogicalType union implementation_status: false compatibility: forward_compatible released_in: "2.4.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L471 + prs: "[#51](https://github.com/apache/parquet-format/pull/51)" version_note: Supersedes `ConvertedType` enumdeprecated in [2.9.0](https://github.com/apache/parquet-format/releases/tag/apache-parquet-format-2.9.0) - id: logical-nanosecond-time-timestamp display_name: Nanosecond TIME/TIMESTAMP @@ -145,10 +160,12 @@ features: compatibility: forward_compatible released_in: "2.6.0" feature_url: https://github.com/apache/parquet-format/blob/96656a543a2165d57cc1c9abefaad7f9aeb563a5/src/main/thrift/parquet.thrift#L352 + prs: "[#102](https://github.com/apache/parquet-format/pull/102)" - id: logical-variant-shredding display_name: Variant shredding implementation_status: false compatibility: forward_compatible released_in: "2.12.0" feature_url: https://github.com/apache/parquet-format/blob/master/VariantShredding.md + prs: "[#456](https://github.com/apache/parquet-format/pull/456), [#461](https://github.com/apache/parquet-format/pull/461)" approval: "[2025-08-24](https://lists.apache.org/thread/obn1yzhgm5zlznwrdpg7f66mswwooxw7)" diff --git a/layouts/shortcodes/format-versions.html b/layouts/shortcodes/format-versions.html index 21e30f6e..95612452 100644 --- a/layouts/shortcodes/format-versions.html +++ b/layouts/shortcodes/format-versions.html @@ -47,8 +47,9 @@ Feature - Released in + Release Source + PRs Approved Notes @@ -79,6 +80,11 @@ {{ $release.source_display }} + + {{- with $feature.prs -}} + {{ . | markdownify }} + {{- end -}} + {{- with $feature.approval -}} {{ . | markdownify }}