Prune unread nested parquet leaves when a projected column is cast to a narrower type (nested schema pruning)

datafusion/common/src/config.rs

@@ -1181,6 +1181,14 @@ config_namespace! {
         /// (reading) Use any available bloom filters when reading parquet files
         pub bloom_filter_on_read: bool, default = true
 
+        /// (reading) If true, when a projected nested column (struct, or
+        /// struct nested in lists) is read through a cast to a narrower
+        /// nested type — as happens when the table schema declares fewer
+        /// struct fields than the parquet file contains — the reader only
+        /// fetches and decodes the leaf columns the cast retains, instead of
+        /// the entire column
+        pub nested_projection_pruning: bool, default = true
+
         /// (reading) The maximum predicate cache size, in bytes. When
         /// `pushdown_filters` is enabled, sets the maximum memory used to cache
         /// the results of predicate evaluation between filter evaluation and

datafusion/common/src/file_options/parquet_writer.rs

@@ -242,6 +242,7 @@ impl ParquetOptions {
             maximum_parallel_row_group_writers: _,
             maximum_buffered_record_batches_per_stream: _,
             bloom_filter_on_read: _, // reads not used for writer props
+            nested_projection_pruning: _, // reads not used for writer props
             schema_force_view_types: _,
             binary_as_string: _, // not used for writer props
             coerce_int96: _,     // not used for writer props
@@ -500,6 +501,7 @@ mod tests {
             maximum_buffered_record_batches_per_stream: defaults
                 .maximum_buffered_record_batches_per_stream,
             bloom_filter_on_read: defaults.bloom_filter_on_read,
+            nested_projection_pruning: defaults.nested_projection_pruning,
             schema_force_view_types: defaults.schema_force_view_types,
             binary_as_string: defaults.binary_as_string,
             skip_arrow_metadata: defaults.skip_arrow_metadata,
@@ -620,6 +622,8 @@ mod tests {
                 maximum_buffered_record_batches_per_stream: global_options_defaults
                     .maximum_buffered_record_batches_per_stream,
                 bloom_filter_on_read: global_options_defaults.bloom_filter_on_read,
+                nested_projection_pruning: global_options_defaults
+                    .nested_projection_pruning,
                 max_predicate_cache_size: global_options_defaults
                     .max_predicate_cache_size,
                 schema_force_view_types: global_options_defaults.schema_force_view_types,

datafusion/core/Cargo.toml

@@ -247,6 +247,11 @@ harness = false
 name = "parquet_struct_query"
 required-features = ["parquet"]
 
+[[bench]]
+harness = false
+name = "parquet_nested_schema_pruning"
+required-features = ["parquet"]
+
 [[bench]]
 harness = false
 name = "parquet_struct_projection"