From 2636affa976f4fa38be1507b1739c5aa8c91eb4d Mon Sep 17 00:00:00 2001 From: Nic Crane Date: Thu, 4 Jun 2026 09:54:30 +0100 Subject: [PATCH] Add note about dictionary not helping --- python/pyarrow/parquet/core.py | 2 ++ r/R/parquet.R | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/python/pyarrow/parquet/core.py b/python/pyarrow/parquet/core.py index ff880fdcf52c..7fed7d09fe21 100644 --- a/python/pyarrow/parquet/core.py +++ b/python/pyarrow/parquet/core.py @@ -769,6 +769,8 @@ def _sanitize_table(table, new_schema, flavor): When encoding the column, if the dictionary size is too large, the column will fallback to ``PLAIN`` encoding. Specially, ``BOOLEAN`` type doesn't support dictionary encoding. + On columns with many unique values, dictionary encoding can increase + file size and reduce compression effectiveness. compression : str or dict, default 'snappy' Specify the compression codec, either on a general basis or per-column. Valid values: {'NONE', 'SNAPPY', 'GZIP', 'BROTLI', 'LZ4', 'LZ4_RAW', 'ZSTD'}. diff --git a/r/R/parquet.R b/r/R/parquet.R index 6415e36b03ce..2fe0fef771f5 100644 --- a/r/R/parquet.R +++ b/r/R/parquet.R @@ -106,7 +106,9 @@ read_parquet <- function( #' @param compression compression algorithm. Default "snappy". See details. #' @param compression_level compression level. Meaning depends on compression #' algorithm -#' @param use_dictionary logical: use dictionary encoding? Default `TRUE` +#' @param use_dictionary logical: use dictionary encoding? Default `TRUE`. +#' On columns with many unique values, dictionary encoding can increase +#' file size and reduce compression effectiveness. #' @param write_statistics logical: include statistics? Default `TRUE` #' @param data_page_size Set a target threshold for the approximate encoded #' size of data pages within a column chunk (in bytes). Default 1 MiB.