From 7636da12185d9c2fb3bf465568c8afe4d071a7fc Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Wed, 24 Dec 2025 10:09:04 +0100 Subject: [PATCH 01/27] Rework distributed planning logic (#259) * Rework distributed planner * Ignore some failing tests that should not be failing * Improve default task estimator repartition * Update test snapshots * Factor out plan_annotator.rs * Add docs * Respond to PR feedback * Make TaskCountAnnotation public and part of the TaskEstimator API * Add plan annotator tests (#263) * Fix typo * Add docs --- benchmarks/src/tpch/run.rs | 2 +- src/common/children_helpers.rs | 18 + src/common/mod.rs | 2 + .../distributed_physical_optimizer_rule.rs | 591 ++------ .../distributed_plan_error.rs | 46 - src/distributed_planner/mod.rs | 11 +- src/distributed_planner/network_boundary.rs | 49 +- src/distributed_planner/plan_annotator.rs | 550 +++++++ src/distributed_planner/task_estimator.rs | 235 ++- src/execution_plans/common.rs | 18 +- src/execution_plans/distributed.rs | 9 +- src/execution_plans/mod.rs | 4 +- src/execution_plans/network_coalesce.rs | 174 +-- src/execution_plans/network_shuffle.rs | 165 +- src/execution_plans/partition_isolator.rs | 110 +- src/lib.rs | 4 +- src/metrics/task_metrics_collector.rs | 25 +- src/metrics/task_metrics_rewriter.rs | 32 +- src/protobuf/distributed_codec.rs | 41 +- src/stage.rs | 14 +- src/test_utils/plans.rs | 4 +- tests/introspection.rs | 11 +- tests/tpch_explain_analyze.rs | 1330 ++++++++--------- tests/tpch_plans_test.rs | 1166 +++++++-------- tests/udfs.rs | 8 +- 25 files changed, 2209 insertions(+), 2410 deletions(-) create mode 100644 src/common/children_helpers.rs delete mode 100644 src/distributed_planner/distributed_plan_error.rs create mode 100644 src/distributed_planner/plan_annotator.rs diff --git a/benchmarks/src/tpch/run.rs b/benchmarks/src/tpch/run.rs index b5f9451a..0eec0c47 100644 --- a/benchmarks/src/tpch/run.rs +++ b/benchmarks/src/tpch/run.rs @@ -324,7 +324,7 @@ impl RunOpt { let mut n_tasks = 0; physical_plan.clone().transform_down(|node| { if let Some(node) = node.as_network_boundary() { - n_tasks += node.input_stage().map(|v| v.tasks.len()).unwrap_or(0) + n_tasks += node.input_stage().tasks.len() } Ok(Transformed::no(node)) })?; diff --git a/src/common/children_helpers.rs b/src/common/children_helpers.rs new file mode 100644 index 00000000..c7b4e3ce --- /dev/null +++ b/src/common/children_helpers.rs @@ -0,0 +1,18 @@ +use datafusion::common::{DataFusionError, plan_err}; +use datafusion::physical_plan::ExecutionPlan; +use std::borrow::Borrow; +use std::sync::Arc; + +pub(crate) fn require_one_child( + children: L, +) -> Result, DataFusionError> +where + L: AsRef<[T]>, + T: Borrow>, +{ + let children = children.as_ref(); + if children.len() != 1 { + return plan_err!("Expected exactly 1 children, got {}", children.len()); + } + Ok(children[0].borrow().clone()) +} diff --git a/src/common/mod.rs b/src/common/mod.rs index ae81b10b..413772a0 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -1,4 +1,6 @@ +mod children_helpers; mod map_last_stream; pub mod ttl_map; +pub(crate) use children_helpers::require_one_child; pub(crate) use map_last_stream::map_last_stream; diff --git a/src/distributed_planner/distributed_physical_optimizer_rule.rs b/src/distributed_planner/distributed_physical_optimizer_rule.rs index e4e815f6..6fa1facf 100644 --- a/src/distributed_planner/distributed_physical_optimizer_rule.rs +++ b/src/distributed_planner/distributed_physical_optimizer_rule.rs @@ -1,75 +1,68 @@ -use crate::distributed_planner::distributed_config::DistributedConfig; -use crate::distributed_planner::distributed_plan_error::get_distribute_plan_err; -use crate::distributed_planner::task_estimator::TaskEstimator; -use crate::distributed_planner::{ - DistributedPlanError, NetworkBoundaryExt, limit_tasks_err, non_distributable_err, +use crate::common::require_one_child; +use crate::distributed_planner::plan_annotator::{ + AnnotatedPlan, RequiredNetworkBoundary, annotate_plan, }; -use crate::execution_plans::{DistributedExec, NetworkCoalesceExec}; -use crate::stage::Stage; -use crate::{ChannelResolver, NetworkShuffleExec, PartitionIsolatorExec}; -use datafusion::common::plan_err; -use datafusion::common::tree_node::TreeNodeRecursion; +use crate::{ + DistributedConfig, DistributedExec, NetworkCoalesceExec, NetworkShuffleExec, TaskEstimator, +}; +use datafusion::common::internal_err; +use datafusion::common::tree_node::{Transformed, TreeNode}; +use datafusion::config::ConfigOptions; use datafusion::error::DataFusionError; -use datafusion::physical_expr::Partitioning; -use datafusion::physical_plan::ExecutionPlanProperties; +use datafusion::physical_optimizer::PhysicalOptimizerRule; use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec; use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; -use datafusion::physical_plan::execution_plan::CardinalityEffect; -use datafusion::physical_plan::joins::{HashJoinExec, PartitionMode}; -use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; -use datafusion::physical_plan::streaming::StreamingTableExec; -use datafusion::{ - common::tree_node::{Transformed, TreeNode}, - config::ConfigOptions, - error::Result, - physical_optimizer::PhysicalOptimizerRule, - physical_plan::{ExecutionPlan, repartition::RepartitionExec}, -}; +use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; +use std::fmt::Debug; +use std::ops::AddAssign; use std::sync::Arc; use uuid::Uuid; /// Physical optimizer rule that inspects the plan, places the appropriate network -/// boundaries and breaks it down into stages that can be executed in a distributed manner. -/// -/// The rule has two steps: +/// boundaries, and breaks it down into stages that can be executed in a distributed manner. /// -/// 1. Inject the appropriate distributed execution nodes in the appropriate places. +/// The rule has three steps: /// -/// This is done by looking at specific nodes in the original plan and enhancing them -/// with new additional nodes: -/// - a [DataSourceExec] is wrapped with a [PartitionIsolatorExec] for exposing just a subset -/// of the [DataSourceExec] partitions to the rest of the plan. -/// - a [CoalescePartitionsExec] is followed by a [NetworkCoalesceExec] so that all tasks in the -/// previous stage collapse into just 1 in the next stage. -/// - a [SortPreservingMergeExec] is followed by a [NetworkCoalesceExec] for the same reasons as -/// above -/// - a [RepartitionExec] with a hash partition is wrapped with a [NetworkShuffleExec] for -/// shuffling data to different tasks. +/// 1. Annotate the plan with [annotate_plan]: adds some annotations to each node about how +/// many distributed tasks should be used in the stage containing them, and whether they +/// need a network boundary below or not. +/// For more information about this step, read [annotate_plan] docs. /// +/// 2. Based on the [AnnotatedPlan] returned by [annotate_plan], place all the appropriate +/// network boundaries ([NetworkShuffleExec] and [NetworkCoalesceExec]) with the task count +/// assignation that the annotations required. After this, the plan is already a distributed +/// executable plan. /// -/// 2. Break down the plan into stages -/// -/// Based on the network boundaries ([NetworkShuffleExec], [NetworkCoalesceExec], ...) placed in -/// the plan by the first step, the plan is divided into stages and tasks are assigned to each -/// stage. -/// -/// This step might decide to not respect the amount of tasks each network boundary is requesting, -/// like when a plan is not parallelizable in different tasks (e.g. a collect left [HashJoinExec]) -/// or when a [DataSourceExec] has not enough partitions to be spread across tasks. +/// 3. Place the [CoalesceBatchesExec] in the appropriate places (just below network boundaries), +/// so that we send fewer and bigger record batches over the wire instead of a lot of small ones. #[derive(Debug, Default)] pub struct DistributedPhysicalOptimizerRule; impl PhysicalOptimizerRule for DistributedPhysicalOptimizerRule { fn optimize( &self, - plan: Arc, + original: Arc, cfg: &ConfigOptions, - ) -> Result> { - // We can only optimize plans that are not already distributed - match distribute_plan(apply_network_boundaries(Arc::clone(&plan), cfg)?)? { - None => Ok(plan), - Some(distributed_plan) => Ok(distributed_plan), + ) -> datafusion::common::Result> { + if original.as_any().is::() { + return Ok(original); + } + + let mut plan = Arc::clone(&original); + if original.output_partitioning().partition_count() > 1 { + plan = Arc::new(CoalescePartitionsExec::new(plan)) } + + let annotated = annotate_plan(plan, cfg)?; + + let mut stage_id = 1; + let distributed = distribute_plan(annotated, cfg, Uuid::new_v4(), &mut stage_id)?; + if stage_id == 1 { + return Ok(original); + } + let distributed = push_down_batch_coalescing(distributed, cfg)?; + + Ok(Arc::new(DistributedExec::new(distributed))) } fn name(&self) -> &str { @@ -81,226 +74,100 @@ impl PhysicalOptimizerRule for DistributedPhysicalOptimizerRule { } } -/// Places the appropriate [NetworkBoundary]s in the plan. It will look for certain nodes in the -/// provided plan and wrap them with their distributed equivalent, for example: -/// - A [RepartitionExec] will be wrapped with a [NetworkShuffleExec] for performing the -/// repartition over the network (shuffling). -/// - A [CoalescePartitionsExec] and a [SortPreservingMergeExec] both coalesce P partitions into -/// one, so a [NetworkCoalesceExec] is injected right below them to also coalesce distributed -/// tasks. -/// - A [DataSourceExec] is wrapped with a [PartitionIsolatorExec] so that each distributed task -/// only executes a certain amount of partitions. -/// -/// The amount of tasks that each injected [NetworkBoundary] will spawn is calculated like this: -/// -/// 1. Leaf nodes have the opportunity to provide an estimation of how many tasks should be employed -/// in the [Stage] that contains them. -/// -/// 2. If a [Stage] contains multiple leaf nodes, and all provide a task count estimation, the -/// biggest is taken. -/// -/// 3. When traversing the plan in a bottom to top fashion, this function looks for nodes that either -/// increase or reduce cardinality. -/// - If there's a node that increases cardinality, the next stage will spawn more tasks than the -/// current one. -/// - If there's a node that reduces cardinality, the next stage will spawn fewer tasks than the -/// current one. -/// -/// 4. While traversing the plan from bottom to top, if a new [NetworkBoundary] needs to be placed, -/// it will spawn as many tasks as the previous stage multiplied by a factor determined by -/// wether the cardinality has increased or not. -/// -/// 5. This is repeated until all the [NetworkBoundary]s are placed. -/// -/// ## Example: -/// -/// Given a plan with 3 stages: -/// -/// ```text -/// ┌─────────────────┐ -/// │ Stage 3 │? tasks -/// └────────▲────────┘ -/// ┌────────┴────────┐ -/// │ Stage 2 │? tasks -/// └────────▲────────┘ -/// ┌────────┴────────┐ -/// │ Stage 1 │? tasks -/// └─────────────────┘ -/// ``` -/// -/// 1. Calculate the number of tasks for a bottom stage based on how much data the leaf nodes -/// (e.g. `DataSourceExec`s) are expected to pull. -/// -/// ```text -/// ┌─────────────────┐ -/// │ Stage 3 │? tasks -/// └────────▲────────┘ -/// ┌────────┴────────┐ -/// │ Stage 2 │? tasks -/// └────────▲────────┘ -/// ┌────────┴────────┐ -/// │ Stage 1 │3 tasks -/// └─────────────────┘ -/// ``` -/// -/// 2. Based on the calculated tasks in the leaf stage (e.g. 3 tasks), calculate the amount of -/// tasks in the next stage. -/// This is done by multiplying the task count by a scale factor every time a node that -/// increments or reduces the cardinality of the data appears, which is information present in -/// the `fn cardinality_effect(&self) -> CardinalityEffect` method. For example, if "Stage 1" -/// has a partial aggregation step, and the scale factor is 1.5, it will look like this: -/// -/// ```text -/// ┌─────────────────┐ -/// │ Stage 3 │? tasks -/// └────────▲────────┘ -/// ┌────────┴────────┐ -/// │ Stage 2 │3/1.5 = 2 tasks -/// └────────▲────────┘ -/// ┌────────┴────────┐ -/// │ Stage 1 │3 tasks (cardinality effect factor of 1.5) -/// └─────────────────┘ -/// ``` -/// -/// -/// 3. This is repeated recursively until all tasks have been assigned to all stages, keeping into -/// account the cardinality effect different nodes in subplans have. If there is no -/// cardinality effect (e.g. `ProjectExec` nodes), then the task count is kept across stages: -/// -/// ```text -/// ┌─────────────────┐ -/// │ Stage 3 │2 tasks -/// └────────▲────────┘ -/// ┌────────┴────────┐ -/// │ Stage 2 │2 tasks -/// └────────▲────────┘ -/// ┌────────┴────────┐ -/// │ Stage 1 │3 tasks -/// └─────────────────┘ -/// ``` -/// -pub fn apply_network_boundaries( - mut plan: Arc, +/// Takes an [AnnotatedPlan] and returns a modified [ExecutionPlan] with all the network boundaries +/// appropriately placed. This step performs the following modifications to the original +/// [ExecutionPlan]: +/// - The leaf nodes are scaled up in parallelism based on the number of distributed tasks in +/// which they are going to run. This is configurable by the user via the [TaskEstimator] trait. +/// - The appropriate network boundaries are placed in the plan depending on how it was annotated, +/// so new nodes like [NetworkCoalesceExec] and [NetworkShuffleExec] will be present. +fn distribute_plan( + annotated_plan: AnnotatedPlan, cfg: &ConfigOptions, -) -> Result> { - if plan.output_partitioning().partition_count() > 1 { - // Coalescing partitions here will allow us to put a NetworkCoalesceExec on top - // of the plan, executing it in parallel. - plan = Arc::new(CoalescePartitionsExec::new(plan)) - } - let distributed_cfg = DistributedConfig::from_config_options(cfg)?; - let urls = distributed_cfg.__private_channel_resolver.0.get_urls()?; - // If there are 1 or 0 available workers, it does not make sense to distribute the query, - // so don't. - if urls.len() <= 1 { - return Ok(plan); + query_id: Uuid, + stage_id: &mut usize, +) -> Result, DataFusionError> { + let d_cfg = DistributedConfig::from_config_options(cfg)?; + + let children = annotated_plan.children; + // This is a leaf node, so we need to scale it up with the final task count. + if children.is_empty() { + let scaled_up = d_cfg.__private_task_estimator.scale_up_leaf_node( + &annotated_plan.plan, + annotated_plan.task_count.as_usize(), + cfg, + ); + return Ok(scaled_up.unwrap_or(annotated_plan.plan)); } - let ctx = _apply_network_boundaries(plan, cfg, urls.len())?; - Ok(ctx.plan) -} -/// [ApplyNetworkBoundariesCtx] helps keeping track of the stage of the task count calculations -/// while recursing through [ExecutionPlan]s. -struct ApplyNetworkBoundariesCtx { - task_count: usize, - this_stage_sf: f64, - next_stage_sf: f64, - plan: Arc, -} + let parent_task_count = annotated_plan.task_count.as_usize(); + let max_child_task_count = children.iter().map(|v| v.task_count.as_usize()).max(); -impl ApplyNetworkBoundariesCtx { - /// Returns the task count with the calculated current scale factor, and swaps the scale - /// factor calculated for the next stage by the current one, resetting the next stage scale - /// factor. - /// - /// This is called whenever a new [NetworkBoundary] is introduced, which marks the end of - /// one [Stage], and the beginning of the next one. - fn scale_task_count_and_swap(&mut self) -> Result { - let task_count = (self.task_count as f64 * self.this_stage_sf).ceil() as usize; - self.this_stage_sf = self.next_stage_sf; - self.next_stage_sf = 1.0; - if task_count == 0 { - return plan_err!( - "Attempted to assign a distributed task count of 0. This should never happen." - ); - } - Ok(task_count) + let new_children = children + .into_iter() + .map(|child| distribute_plan(child, cfg, query_id, stage_id)) + .collect::, _>>()?; + + // It does not need a NetworkBoundary, so just keep recursing. + let Some(nb_req) = annotated_plan.required_network_boundary else { + return annotated_plan.plan.with_new_children(new_children); + }; + + // It would need a network boundary, but on both sides of the boundary there is just 1 task, + // so we are fine with not introducing any network boundary. + if parent_task_count == 1 && max_child_task_count == Some(1) { + return annotated_plan.plan.with_new_children(new_children); } - /// Scale the tasks count of the next stage. Note that, even if nodes in the current stage scale - /// up or down the cardinality, that doesn't affect the task count for the current stage, but - /// for the next one, as that's the one that will see the benefits of the current stage - /// compressing the amount of data flowing. - fn apply_scale_factor(&mut self, sf: f64) { - match self.plan.cardinality_effect() { - CardinalityEffect::LowerEqual => self.next_stage_sf /= sf, - CardinalityEffect::GreaterEqual => self.next_stage_sf *= sf, - _ => {} - } + // If the current node has a RepartitionExec below, it needs a shuffle, so put one + // NetworkShuffleExec boundary in between the RepartitionExec and the current node. + if nb_req == RequiredNetworkBoundary::Shuffle { + let new_child = Arc::new(NetworkShuffleExec::try_new( + require_one_child(new_children)?, + query_id, + *stage_id, + parent_task_count, + max_child_task_count.unwrap_or(1), + )?); + stage_id.add_assign(1); + return annotated_plan.plan.with_new_children(vec![new_child]); } + + // If this is a CoalescePartitionsExec or a SortMergePreservingExec, it means that the original + // plan is trying to merge all partitions into one. We need to go one step ahead and also merge + // all distributed tasks into one. + if nb_req == RequiredNetworkBoundary::Coalesce { + let new_child = Arc::new(NetworkCoalesceExec::try_new( + require_one_child(new_children)?, + query_id, + *stage_id, + parent_task_count, + max_child_task_count.unwrap_or(1), + )?); + stage_id.add_assign(1); + return annotated_plan.plan.with_new_children(vec![new_child]); + } + + internal_err!( + "Unreachable code reached in distribute_plan. Could not determine how to place a network boundary below {}", + annotated_plan.plan.name() + ) } -fn _apply_network_boundaries( +/// Rearranges the [CoalesceBatchesExec] nodes in the plan so that they are placed right below +/// the network boundaries, so that fewer but bigger record batches are sent over the wire across +/// stages. +fn push_down_batch_coalescing( plan: Arc, cfg: &ConfigOptions, - max_tasks: usize, -) -> Result { - let mut ctx = None; - - let children = plan.children(); - let mut new_children = Vec::with_capacity(children.len()); - // Recurse now in to the children so that nodes are the bottom are evaluated first. - for child in children.iter() { - let prev_ctx = _apply_network_boundaries(Arc::clone(child), cfg, max_tasks)?; - new_children.push(Arc::clone(&prev_ctx.plan)); - match &mut ctx { - None => { - ctx.replace(ApplyNetworkBoundariesCtx { - task_count: prev_ctx.task_count, - this_stage_sf: prev_ctx.this_stage_sf, - next_stage_sf: prev_ctx.next_stage_sf, - plan: Arc::clone(&plan), - }); - } - Some(ctx) => { - ctx.task_count = ctx.task_count.max(prev_ctx.task_count).min(max_tasks); - ctx.next_stage_sf = ctx.next_stage_sf.max(prev_ctx.next_stage_sf); - } - } - } - +) -> Result, DataFusionError> { let d_cfg = DistributedConfig::from_config_options(cfg)?; - let Some(mut ctx) = ctx else { - // As there is no context, it means that children.is_empty() == true and no ctx was set, so - // this is a leaf node, maybe a DataSourceExec, or maybe something else custom from the - // user. We need to estimate how many tasks are needed for this leaf node, as we'll use - // that for choosing the amount of tasks in upper stages. - let Some(estimate) = d_cfg.__private_task_estimator.estimate_tasks(&plan, cfg) else { - // We could not determine how many tasks this leaf node should run on, so - // assume it cannot be distributed and employ just 1 task. - return Ok(ApplyNetworkBoundariesCtx { - task_count: 1, - this_stage_sf: 1.0, - next_stage_sf: 1.0, - plan, - }); - }; - return Ok(ApplyNetworkBoundariesCtx { - task_count: estimate.task_count, - this_stage_sf: 1.0, - next_stage_sf: 1.0, - plan: estimate.new_plan.unwrap_or(plan), - }); - }; - ctx.plan = ctx.plan.with_new_children(new_children)?; + let transformed = plan.transform_up(|plan| { + let Some(node) = plan.as_any().downcast_ref::() else { + return Ok(Transformed::no(plan)); + }; - // If this is a hash RepartitionExec, introduce a shuffle. - if let Some(node) = ctx.plan.as_any().downcast_ref::() { - if !matches!(node.partitioning(), Partitioning::Hash(_, _)) { - return Ok(ctx); - } - let task_count = ctx.scale_task_count_and_swap()?; // Network shuffles imply partitioning each data stream in a lot of different partitions, // which means that each resulting stream might contain tiny batches. It's important to // have decent sized batches here as this will ultimately be sent over the wire, and the @@ -310,195 +177,20 @@ fn _apply_network_boundaries( // CoalesceBatchesExec, we just need to tell RepartitionExec to output a // `d_cfg.shuffle_batch_size` batch size. // Tracked by https://github.com/datafusion-contrib/datafusion-distributed/issues/243 - if d_cfg.shuffle_batch_size > 0 { - ctx.plan = Arc::new(CoalesceBatchesExec::new(ctx.plan, d_cfg.shuffle_batch_size)); - } - ctx.plan = Arc::new(NetworkShuffleExec::try_new(ctx.plan, task_count)?); - return Ok(ctx); - } else if let Some(coalesce_batches) = ctx.plan.as_any().downcast_ref::() { - // If the batch coalescing is before the network boundary, remove it, as we don't - // want it there, we want it after, and the code that adds it lives just some lines above. - if coalesce_batches.input().is_network_boundary() { - ctx.plan = Arc::clone(coalesce_batches.input()); - } - } - - // If this is a CoalescePartitionsExec, it means that the original plan is trying to - // merge all partitions into one. We need to go one step ahead and also merge all tasks - // into one. - if let Some(node) = ctx.plan.as_any().downcast_ref::() { - let input = Arc::clone(node.input()); - // If the immediate child is a PartitionIsolatorExec, it means that the rest of the - // plan is just a couple of non-computational nodes that are probably not worth - // distributing. - if input.as_any().is::() { - return Ok(ctx); - } - - let task_count = ctx.scale_task_count_and_swap()?; - let new_child = NetworkCoalesceExec::new(input, task_count); - ctx.plan = ctx.plan.with_new_children(vec![Arc::new(new_child)])?; - return Ok(ctx); - } - - // The SortPreservingMergeExec node will try to coalesce all partitions into just 1. - // We need to account for it and help it by also coalescing all tasks into one, therefore - // a NetworkCoalesceExec is introduced. - if let Some(node) = ctx.plan.as_any().downcast_ref::() { - let input = Arc::clone(node.input()); - - let task_count = ctx.scale_task_count_and_swap()?; - let new_child = NetworkCoalesceExec::new(input, task_count); - ctx.plan = ctx.plan.with_new_children(vec![Arc::new(new_child)])?; - return Ok(ctx); - } - - // upscales or downscales the task count factor of the next stage depending on the - // cardinality of the current plan. - ctx.apply_scale_factor(d_cfg.cardinality_task_count_factor); - - Ok(ctx) -} - -/// Takes a plan with certain network boundaries in it ([NetworkShuffleExec], [NetworkCoalesceExec], ...) -/// and breaks it down into stages. -/// -/// This can be used a standalone function for distributing arbitrary plans in which users have -/// manually placed network boundaries, or as part of the [DistributedPhysicalOptimizerRule] that -/// places the network boundaries automatically as a standard [PhysicalOptimizerRule]. -/// -/// If there's nothing to distribute in the plan, the function returns None. -pub fn distribute_plan( - plan: Arc, -) -> Result>, DataFusionError> { - if plan.as_any().is::() { - return Ok(Some(plan)); - } - let stage = match _distribute_plan_inner(Uuid::new_v4(), plan.clone(), &mut 1, 0, 1) { - Ok(stage) => stage, - Err(err) => { - return match get_distribute_plan_err(&err) { - Some(DistributedPlanError::NonDistributable(_)) => plan - .transform_down(|plan| { - // If the node cannot be distributed, rollback all the network boundaries. - if let Some(nb) = plan.as_network_boundary() { - return Ok(Transformed::yes(nb.rollback()?)); - } - Ok(Transformed::no(plan)) - }) - .map(|v| Some(v.data)), - _ => Err(err), - }; - } - }; - // After running the distributed planner, only 1 stage was created, meaning that the plan - // was not distributed. - if stage.num == 1 { - return Ok(None); - } - let plan = stage.plan.decoded()?; - Ok(Some(Arc::new(DistributedExec::new(Arc::clone(plan))))) -} - -fn _distribute_plan_inner( - query_id: Uuid, - plan: Arc, - num: &mut usize, - depth: usize, - n_tasks: usize, -) -> Result { - let mut distributed = plan.clone().transform_down(|plan| { - // We cannot break down CollectLeft hash joins into more than 1 task, as these need - // a full materialized build size with all the data in it. - // - // Maybe in the future these can be broadcast joins? - if let Some(node) = plan.as_any().downcast_ref::() { - if n_tasks > 1 && node.mode == PartitionMode::CollectLeft { - return Err(limit_tasks_err(1)); - } - } - - // We cannot distribute [StreamingTableExec] nodes, so abort distribution. - if plan.as_any().is::() { - return Err(non_distributable_err(StreamingTableExec::static_name())) - } - - if let Some(node) = plan.as_any().downcast_ref::() { - // If there's only 1 task, no need to perform any isolation. - if n_tasks == 1 { - return Ok(Transformed::yes(Arc::clone(plan.children().first().unwrap()))); - } - let node = node.ready(n_tasks)?; - return Ok(Transformed::new(Arc::new(node), true, TreeNodeRecursion::Jump)); - } - - let Some(mut dnode) = plan.as_network_boundary().map(Referenced::Borrowed) else { + let Some(shuffle) = node.input().as_any().downcast_ref::() else { return Ok(Transformed::no(plan)); }; - - let stage = loop { - let input_stage_info = dnode.as_ref().get_input_stage_info(n_tasks)?; - // If the current stage has just 1 task, and the next stage is only going to have - // 1 task, there's no point in having a network boundary in between, they can just - // communicate in memory. - if n_tasks == 1 && input_stage_info.task_count == 1 { - let mut n = dnode.as_ref().rollback()?; - if let Some(node) = n.as_any().downcast_ref::() { - // Also trim PartitionIsolatorExec out of the plan. - n = Arc::clone(node.children().first().unwrap()); - } - return Ok(Transformed::yes(n)); - } - match _distribute_plan_inner(query_id, input_stage_info.plan, num, depth + 1, input_stage_info.task_count) { - Ok(v) => break v, - Err(e) => match get_distribute_plan_err(&e) { - None => return Err(e), - Some(DistributedPlanError::LimitTasks(limit)) => { - // While attempting to build a new stage, a failure was raised stating - // that no more than `limit` tasks can be used for it, so we are going - // to limit the amount of tasks to the requested number and try building - // the stage again. - if input_stage_info.task_count == *limit { - return plan_err!("A node requested {limit} tasks for the stage its in, but that stage already has that many tasks"); - } - dnode = Referenced::Arced(dnode.as_ref().with_input_task_count(*limit)?); - } - Some(DistributedPlanError::NonDistributable(_)) => { - // This full plan is non-distributable, so abort any task and stage - // assignation. - return Err(e); - } - }, - } - }; - let node = dnode.as_ref().with_input_stage(stage)?; - Ok(Transformed::new(node, true, TreeNodeRecursion::Jump)) + // First the child of the NetworkShuffleExec. + let plan = shuffle.input_stage.plan.decoded()?; + // Then a CoalesceBatchesExec for sending bigger chunks over the wire. + let plan = CoalesceBatchesExec::new(Arc::clone(plan), d_cfg.shuffle_batch_size); + // Then the NetworkShuffleExec itself with the CoalesceBatchesExec as a child. + let plan = Arc::clone(node.input()).with_new_children(vec![Arc::new(plan)])?; + + Ok(Transformed::yes(plan)) })?; - // The head stage is executable, and upon execution, it will lazily assign worker URLs to - // all tasks. This must only be done once, so the executable StageExec must only be called - // once on 1 partition. - if depth == 0 && distributed.data.output_partitioning().partition_count() > 1 { - distributed.data = Arc::new(CoalescePartitionsExec::new(distributed.data)); - } - - let stage = Stage::new(query_id, *num, distributed.data, n_tasks); - *num += 1; - Ok(stage) -} -/// Helper enum for storing either borrowed or owned trait object references -enum Referenced<'a, T: ?Sized> { - Borrowed(&'a T), - Arced(Arc), -} - -impl Referenced<'_, T> { - fn as_ref(&self) -> &T { - match self { - Self::Borrowed(r) => r, - Self::Arced(arc) => arc.as_ref(), - } - } + Ok(transformed.data) } #[cfg(test)] @@ -511,7 +203,7 @@ mod tests { use datafusion::prelude::{SessionConfig, SessionContext}; use itertools::Itertools; use std::sync::Arc; - /* shema for the "weather" table + /* schema for the "weather" table MinTemp [type=DOUBLE] [repetitiontype=OPTIONAL] MaxTemp [type=DOUBLE] [repetitiontype=OPTIONAL] @@ -871,12 +563,11 @@ mod tests { }) .await; assert_snapshot!(plan, @r" - CoalescePartitionsExec - ProjectionExec: expr=[table_catalog@0 as table_catalog, table_schema@1 as table_schema, table_name@2 as table_name, column_name@3 as column_name, data_type@5 as data_type, is_nullable@4 as is_nullable] - CoalesceBatchesExec: target_batch_size=8192 - FilterExec: table_name@2 = weather - RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 - StreamingTableExec: partition_sizes=1, projection=[table_catalog, table_schema, table_name, column_name, is_nullable, data_type] + ProjectionExec: expr=[table_catalog@0 as table_catalog, table_schema@1 as table_schema, table_name@2 as table_name, column_name@3 as column_name, data_type@5 as data_type, is_nullable@4 as is_nullable] + CoalesceBatchesExec: target_batch_size=8192 + FilterExec: table_name@2 = weather + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + StreamingTableExec: partition_sizes=1, projection=[table_catalog, table_schema, table_name, column_name, is_nullable, data_type] "); } diff --git a/src/distributed_planner/distributed_plan_error.rs b/src/distributed_planner/distributed_plan_error.rs deleted file mode 100644 index 2b15dc49..00000000 --- a/src/distributed_planner/distributed_plan_error.rs +++ /dev/null @@ -1,46 +0,0 @@ -use datafusion::common::DataFusionError; -use std::error::Error; -use std::fmt::{Display, Formatter}; - -/// Error thrown during distributed planning that prompts the planner to change something and -/// try again. -#[derive(Debug)] -pub enum DistributedPlanError { - /// Prompts the planner to limit the amount of tasks used in the stage that is currently - /// being planned. - LimitTasks(usize), - /// Signals the planner that this whole plan is non-distributable. This can happen if - /// certain nodes are present, like `StreamingTableExec`, which are typically used in - /// queries that rather performing some execution, they perform some introspection. - NonDistributable(&'static str), -} - -impl Display for DistributedPlanError { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - DistributedPlanError::LimitTasks(n) => write!(f, "LimitTasksErr: {n}"), - DistributedPlanError::NonDistributable(name) => write!(f, "NonDistributable: {name}"), - } - } -} - -impl Error for DistributedPlanError {} - -/// Builds a [DistributedPlanError::LimitTasks] error. This error prompts the distributed planner -/// to try rebuilding the current stage with a limited amount of tasks. -pub fn limit_tasks_err(limit: usize) -> DataFusionError { - DataFusionError::External(Box::new(DistributedPlanError::LimitTasks(limit))) -} - -/// Builds a [DistributedPlanError::NonDistributable] error. This error prompts the distributed -/// planner to not distribute the query at all. -pub fn non_distributable_err(name: &'static str) -> DataFusionError { - DataFusionError::External(Box::new(DistributedPlanError::NonDistributable(name))) -} - -pub(crate) fn get_distribute_plan_err(err: &DataFusionError) -> Option<&DistributedPlanError> { - let DataFusionError::External(err) = err else { - return None; - }; - err.downcast_ref() -} diff --git a/src/distributed_planner/mod.rs b/src/distributed_planner/mod.rs index 3f80d2d0..9b99678b 100644 --- a/src/distributed_planner/mod.rs +++ b/src/distributed_planner/mod.rs @@ -1,14 +1,11 @@ mod distributed_config; mod distributed_physical_optimizer_rule; -mod distributed_plan_error; mod network_boundary; +mod plan_annotator; mod task_estimator; pub use distributed_config::DistributedConfig; -pub use distributed_physical_optimizer_rule::{ - DistributedPhysicalOptimizerRule, apply_network_boundaries, distribute_plan, -}; -pub use distributed_plan_error::{DistributedPlanError, limit_tasks_err, non_distributable_err}; -pub use network_boundary::{InputStageInfo, NetworkBoundary, NetworkBoundaryExt}; +pub use distributed_physical_optimizer_rule::DistributedPhysicalOptimizerRule; +pub use network_boundary::{NetworkBoundary, NetworkBoundaryExt}; pub(crate) use task_estimator::set_distributed_task_estimator; -pub use task_estimator::{TaskEstimation, TaskEstimator}; +pub use task_estimator::{TaskCountAnnotation, TaskEstimation, TaskEstimator}; diff --git a/src/distributed_planner/network_boundary.rs b/src/distributed_planner/network_boundary.rs index 8053a532..25359724 100644 --- a/src/distributed_planner/network_boundary.rs +++ b/src/distributed_planner/network_boundary.rs @@ -1,43 +1,11 @@ use crate::{NetworkCoalesceExec, NetworkShuffleExec, Stage}; -use datafusion::common::plan_err; use datafusion::physical_plan::ExecutionPlan; use std::sync::Arc; -/// Necessary information for building a [Stage] during distributed planning. -/// -/// [NetworkBoundary]s return this piece of data so that the distributed planner know how to -/// build the next [Stage] from which the [NetworkBoundary] is going to receive data. -/// -/// Some network boundaries might perform some modifications in their children, like scaling -/// up the number of partitions, or injecting a specific [ExecutionPlan] on top. -pub struct InputStageInfo { - /// The head plan of the [Stage] that is about to be built. - pub plan: Arc, - /// The amount of tasks the [Stage] will have. - pub task_count: usize, -} - /// This trait represents a node that introduces the necessity of a network boundary in the plan. /// The distributed planner, upon stepping into one of these, will break the plan and build a stage /// out of it. pub trait NetworkBoundary: ExecutionPlan { - /// Returns the information necessary for building the next stage from which this - /// [NetworkBoundary] is going to collect data. - fn get_input_stage_info(&self, task_count: usize) - -> datafusion::common::Result; - - /// re-assigns a different number of input tasks to the current [NetworkBoundary]. - /// - /// This will be called if upon building a stage, a [crate::distributed_planner::distributed_physical_optimizer_rule::DistributedPlanError::LimitTasks] error - /// is returned, prompting the [NetworkBoundary] to choose a different number of input tasks. - fn with_input_task_count( - &self, - input_tasks: usize, - ) -> datafusion::common::Result>; - - /// Returns the input tasks assigned to this [NetworkBoundary]. - fn input_task_count(&self) -> usize; - /// Called when a [Stage] is correctly formed. The [NetworkBoundary] can use this /// information to perform any internal transformations necessary for distributed execution. /// @@ -48,22 +16,7 @@ pub trait NetworkBoundary: ExecutionPlan { ) -> datafusion::common::Result>; /// Returns the assigned input [Stage], if any. - fn input_stage(&self) -> Option<&Stage>; - - /// The planner might decide to remove this [NetworkBoundary] from the plan if it decides that - /// it's not going to bring any benefit. The [NetworkBoundary] will be replaced with whatever - /// this function returns. - fn rollback(&self) -> datafusion::common::Result> { - let children = self.children(); - if children.len() != 1 { - return plan_err!( - "Expected distributed node {} to have exactly 1 children, but got {}", - self.name(), - children.len() - ); - } - Ok(Arc::clone(children.first().unwrap())) - } + fn input_stage(&self) -> &Stage; } /// Extension trait for downcasting dynamic types to [NetworkBoundary]. diff --git a/src/distributed_planner/plan_annotator.rs b/src/distributed_planner/plan_annotator.rs new file mode 100644 index 00000000..f3182bcb --- /dev/null +++ b/src/distributed_planner/plan_annotator.rs @@ -0,0 +1,550 @@ +use crate::{DistributedConfig, TaskCountAnnotation, TaskEstimator}; +use datafusion::common::{DataFusionError, plan_datafusion_err}; +use datafusion::config::ConfigOptions; +use datafusion::physical_expr::Partitioning; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; +use datafusion::physical_plan::execution_plan::CardinalityEffect; +use datafusion::physical_plan::joins::{HashJoinExec, PartitionMode}; +use datafusion::physical_plan::repartition::RepartitionExec; +use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; +use std::fmt::{Debug, Formatter}; +use std::sync::Arc; + +/// Annotation attached to a single [ExecutionPlan] that determines the kind of network boundary +/// needed just below itself. +#[derive(Debug, PartialEq)] +pub(super) enum RequiredNetworkBoundary { + Shuffle, + Coalesce, +} + +/// Wraps an [ExecutionPlan] and annotates it with information about how many distributed tasks +/// it should run on, and whether it needs a network boundary below or not. +pub(super) struct AnnotatedPlan { + /// The annotated [ExecutionPlan]. + pub(super) plan: Arc, + /// The annotated children of this [ExecutionPlan]. This will always hold the same nodes as + /// `self.plan.children()` but annotated. + pub(super) children: Vec, + + // annotation fields + /// How many distributed tasks this plan should run on. + pub(super) task_count: TaskCountAnnotation, + /// Whether this [ExecutionPlan] needs a network boundary below it or not. Even if this is set + /// to `Some()`, a later step can still decide to not place the network boundary under certain + /// situations, like if both sides of the boundary have a task count equal to 1. + pub(super) required_network_boundary: Option, +} + +impl Debug for AnnotatedPlan { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt_dbg(f: &mut Formatter<'_>, plan: &AnnotatedPlan, depth: usize) -> std::fmt::Result { + write!( + f, + "{}{}: task_count={:?}", + " ".repeat(depth * 2), + plan.plan.name(), + plan.task_count + )?; + if let Some(nb) = &plan.required_network_boundary { + write!(f, ", required_network_boundary={nb:?}")?; + } + writeln!(f)?; + for child in plan.children.iter() { + fmt_dbg(f, child, depth + 1)?; + } + Ok(()) + } + + fmt_dbg(f, self, 0) + } +} + +/// Annotates recursively an [ExecutionPlan] and its children with information about how many +/// distributed tasks it should run on, and whether it needs a network boundary below it or not. +/// +/// This is the first step of the distribution process, where the plan structure is still left +/// untouched and the existing nodes are just annotated for future steps to perform the distribution. +/// +/// The plans are annotated in a bottom-to-top manner, starting with the leaf nodes all the way +/// to the head of the plan: +/// +/// 1. Leaf nodes have the opportunity to provide an estimation of how many distributed tasks should +/// be used for the whole stage that will execute them. +/// +/// 2. If a stage contains multiple leaf nodes, and all provide a task count estimation, the +/// biggest is taken. +/// +/// 3. When traversing the plan in a bottom-to-top fashion, this function looks for nodes that +/// either increase or reduce cardinality: +/// - If there's a node that increases cardinality, the next stage will spawn more tasks than +/// the current one. +/// - If there's a node that reduces cardinality, the next stage will spawn fewer tasks than the +/// current one. +/// +/// 4. At a certain point, the function will reach a node that needs a network boundary below; in +/// that case, the node is annotated with a [RequiredNetworkBoundary] value. At this point, all +/// the nodes below must reach a consensus about the final task count for the stage below the +/// network boundary. +/// +/// 5. This process is repeated recursively until all nodes are annotated. +/// +/// ## Example: +/// +/// Following the process above, an annotated plan will look like this: +/// +/// ```text +/// ┌────────────────────┐ task_count: Maximum(1) (because we try to coalesce all partitions into 1) +/// │ CoalescePartitions │ network_boundary: Some(Coalesce) +/// └──────────▲─────────┘ +/// │ +/// ┌──────────┴─────────┐ task_count: Desired(3) (inherited from the child) +/// │ Projection │ network_boundary: None +/// └──────────▲─────────┘ +/// │ +/// ┌──────────┴─────────┐ task_count: Desired(3) (as this node requires a network boundary below, +/// │ Aggregation │ and the stage below reduces the cardinality of the data because of the +/// │ (final) │ partial aggregation, we can choose a smaller amount of tasks) +/// └──────────▲─────────┘ network_boundary: Some(Shuffle) (because the child is a repartition) +/// │ +/// ┌──────────┴─────────┐ task_count: Desired(4) (inherited from the child) +/// │ Repartition │ network_boundary: None +/// └──────────▲─────────┘ +/// │ +/// ┌──────────┴─────────┐ task_count: Desired(4) (inherited from the child) +/// │ Aggregation │ network_boundary: None +/// │ (partial) │ +/// └──────────▲─────────┘ +/// │ +/// ┌──────────┴─────────┐ task_count: Desired(4) (this was set by a TaskEstimator implementation) +/// │ DataSourceExec │ network_boundary: None +/// └────────────────────┘ +/// ``` +/// +/// ``` +pub(super) fn annotate_plan( + plan: Arc, + cfg: &ConfigOptions, +) -> Result { + _annotate_plan(plan, cfg, true) +} +fn _annotate_plan( + plan: Arc, + cfg: &ConfigOptions, + root: bool, +) -> Result { + use TaskCountAnnotation::*; + let d_cfg = DistributedConfig::from_config_options(cfg)?; + let n_workers = d_cfg.__private_channel_resolver.0.get_urls()?.len().max(1); + + let annotated_children = plan + .children() + .iter() + .map(|child| _annotate_plan(Arc::clone(child), cfg, false)) + .collect::, _>>()?; + + if plan.children().is_empty() { + // This is a leaf node, maybe a DataSourceExec, or maybe something else custom from the + // user. We need to estimate how many tasks are needed for this leaf node, and we'll take + // this decision into account when deciding how many tasks will be actually used. + let estimator = &d_cfg.__private_task_estimator; + if let Some(estimate) = estimator.tasks_for_leaf_node(&plan, cfg) { + return Ok(AnnotatedPlan { + plan, + children: Vec::new(), + task_count: estimate.task_count.limit(n_workers), + required_network_boundary: None, + }); + } else { + // We could not determine how many tasks this leaf node should run on, so + // assume it cannot be distributed and used just 1 task. + return Ok(AnnotatedPlan { + plan, + children: Vec::new(), + task_count: Maximum(1), + required_network_boundary: None, + }); + } + } + + // The task count for this plan is decided by the biggest task count from the children; unless + // a child specifies a maximum task count, in that case, the maximum is respected. Some + // nodes can only run in one task. If there is a subplan with a single node declaring that + // it can only run in one task, all the rest of the nodes in the stage need to respect it. + let mut task_count = Desired(1); + for annotated_child in annotated_children.iter() { + task_count = match (task_count, &annotated_child.task_count) { + (Desired(desired), Desired(child)) => Desired(desired.max(*child)), + (Maximum(max), Desired(_)) => Maximum(max), + (Desired(_), Maximum(max)) => Maximum(*max), + (Maximum(max_1), Maximum(max_2)) => Maximum(max_1.min(*max_2)), + }; + task_count = task_count.limit(n_workers); + } + + // We cannot distribute CollectLeft HashJoinExec nodes yet. Once + // https://github.com/datafusion-contrib/datafusion-distributed/pull/229 lands, + // we can remove this check. + if let Some(node) = plan.as_any().downcast_ref::() { + if node.mode == PartitionMode::CollectLeft { + task_count = Maximum(1); + } + } + + // The plan does not need a NetworkBoundary, so just take the biggest task count from + // the children and annotate the plan with that. + let mut annotated_plan = AnnotatedPlan { + required_network_boundary: required_network_boundary_below(plan.as_ref()), + children: annotated_children, + task_count, + plan, + }; + if !(root || annotated_plan.required_network_boundary.is_some()) { + return Ok(annotated_plan); + }; + + // The plan needs a NetworkBoundary. At this point we have all the info we need for choosing + // the right size for the stage below, so what we need to do is take the calculated final + // task count and propagate to all the children that will eventually be part of the stage. + fn propagate_task_count(plan: &mut AnnotatedPlan, task_count: &TaskCountAnnotation) { + plan.task_count = task_count.clone(); + if plan.required_network_boundary.is_none() { + for child in &mut plan.children { + propagate_task_count(child, task_count); + } + } + } + + if let Some(nb) = &annotated_plan.required_network_boundary { + // The plan is a network boundary, so everything below it belongs to the same stage. This + // means that we need to propagate the task count to all the nodes in that stage. + for annotated_child in annotated_plan.children.iter_mut() { + propagate_task_count(annotated_child, &annotated_plan.task_count); + } + + // If the current plan that needs a NetworkBoundary boundary below is either a + // CoalescePartitionsExec or a SortPreservingMergeExec, then we are sure that all the stage + // that they are going to be part of needs to run in exactly one task. + if nb == &RequiredNetworkBoundary::Coalesce { + annotated_plan.task_count = Maximum(1); + return Ok(annotated_plan); + } + + // From now and up in the plan, a new task count needs to be calculated for the next stage. + // Depending on the number of nodes that reduce/increase cardinality, the task count will be + // calculated based on the previous task count multiplied by a factor. + fn calculate_scale_factor(plan: &AnnotatedPlan, f: f64) -> f64 { + let mut sf = None; + + if plan.required_network_boundary.is_none() { + for plan in plan.children.iter() { + sf = match sf { + None => Some(calculate_scale_factor(plan, f)), + Some(sf) => Some(sf.max(calculate_scale_factor(plan, f))), + } + } + } + + let sf = sf.unwrap_or(1.0); + match plan.plan.cardinality_effect() { + CardinalityEffect::LowerEqual => sf / f, + CardinalityEffect::GreaterEqual => sf * f, + _ => sf, + } + } + let sf = calculate_scale_factor( + annotated_plan.children.first().ok_or_else(|| { + plan_datafusion_err!("missing child in a plan annotated with a network boundary") + })?, + d_cfg.cardinality_task_count_factor, + ); + let prev_task_count = annotated_plan.task_count.as_usize() as f64; + annotated_plan.task_count = Desired((prev_task_count * sf).ceil() as usize); + Ok(annotated_plan) + } else if root { + // If this is the root node, it means that we have just finished annotating nodes for the + // subplan belonging to the head stage, so propagate the task count to all children. + let task_count = annotated_plan.task_count.clone(); + propagate_task_count(&mut annotated_plan, &task_count); + Ok(annotated_plan) + } else { + // If this is not the root node, and it's also not a network boundary, then we don't need + // to do anything else. + Ok(annotated_plan) + } +} + +/// Returns if the [ExecutionPlan] requires a network boundary below it, and if it does, the kind +/// of network boundary ([RequiredNetworkBoundary]). +fn required_network_boundary_below(parent: &dyn ExecutionPlan) -> Option { + let children = parent.children(); + let first_child = children.first()?; + + if let Some(r_exec) = first_child.as_any().downcast_ref::() { + if matches!(r_exec.partitioning(), Partitioning::Hash(_, _)) { + return Some(RequiredNetworkBoundary::Shuffle); + } + } + if parent.as_any().is::() + || parent.as_any().is::() + { + // If the next node is a leaf node, distributing this is going to be a bit wasteful, so + // we don't want to do it. + if first_child.children().is_empty() { + return None; + } + return Some(RequiredNetworkBoundary::Coalesce); + } + + None +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::in_memory_channel_resolver::InMemoryChannelResolver; + use crate::test_utils::parquet::register_parquet_tables; + use crate::{DistributedExt, assert_snapshot}; + use datafusion::execution::SessionStateBuilder; + use datafusion::prelude::{SessionConfig, SessionContext}; + use itertools::Itertools; + + /* schema for the "weather" table + + MinTemp [type=DOUBLE] [repetitiontype=OPTIONAL] + MaxTemp [type=DOUBLE] [repetitiontype=OPTIONAL] + Rainfall [type=DOUBLE] [repetitiontype=OPTIONAL] + Evaporation [type=DOUBLE] [repetitiontype=OPTIONAL] + Sunshine [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + WindGustDir [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + WindGustSpeed [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + WindDir9am [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + WindDir3pm [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + WindSpeed9am [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + WindSpeed3pm [type=INT64] [convertedtype=INT_64] [repetitiontype=OPTIONAL] + Humidity9am [type=INT64] [convertedtype=INT_64] [repetitiontype=OPTIONAL] + Humidity3pm [type=INT64] [convertedtype=INT_64] [repetitiontype=OPTIONAL] + Pressure9am [type=DOUBLE] [repetitiontype=OPTIONAL] + Pressure3pm [type=DOUBLE] [repetitiontype=OPTIONAL] + Cloud9am [type=INT64] [convertedtype=INT_64] [repetitiontype=OPTIONAL] + Cloud3pm [type=INT64] [convertedtype=INT_64] [repetitiontype=OPTIONAL] + Temp9am [type=DOUBLE] [repetitiontype=OPTIONAL] + Temp3pm [type=DOUBLE] [repetitiontype=OPTIONAL] + RainToday [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + RISK_MM [type=DOUBLE] [repetitiontype=OPTIONAL] + RainTomorrow [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + */ + + #[tokio::test] + async fn test_select_all() { + let query = r#" + SELECT * FROM weather + "#; + let annotated = sql_to_annotated(query).await; + assert_snapshot!(annotated, @"DataSourceExec: task_count=Desired(3)") + } + + #[tokio::test] + async fn test_aggregation() { + let query = r#" + SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*) + "#; + let annotated = sql_to_annotated(query).await; + assert_snapshot!(annotated, @r" + ProjectionExec: task_count=Maximum(1) + SortPreservingMergeExec: task_count=Maximum(1), required_network_boundary=Coalesce + SortExec: task_count=Desired(2) + ProjectionExec: task_count=Desired(2) + AggregateExec: task_count=Desired(2) + CoalesceBatchesExec: task_count=Desired(2), required_network_boundary=Shuffle + RepartitionExec: task_count=Desired(3) + RepartitionExec: task_count=Desired(3) + AggregateExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + ") + } + + #[tokio::test] + async fn test_left_join() { + let query = r#" + SELECT a."MinTemp", b."MaxTemp" FROM weather a LEFT JOIN weather b ON a."RainToday" = b."RainToday" + "#; + let annotated = sql_to_annotated(query).await; + assert_snapshot!(annotated, @r" + CoalesceBatchesExec: task_count=Maximum(1) + HashJoinExec: task_count=Maximum(1) + CoalescePartitionsExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + ") + } + + #[tokio::test] + async fn test_left_join_distributed() { + let query = r#" + WITH a AS ( + SELECT + AVG("MinTemp") as "MinTemp", + "RainTomorrow" + FROM weather + WHERE "RainToday" = 'yes' + GROUP BY "RainTomorrow" + ), b AS ( + SELECT + AVG("MaxTemp") as "MaxTemp", + "RainTomorrow" + FROM weather + WHERE "RainToday" = 'no' + GROUP BY "RainTomorrow" + ) + SELECT + a."MinTemp", + b."MaxTemp" + FROM a + LEFT JOIN b + ON a."RainTomorrow" = b."RainTomorrow" + "#; + let annotated = sql_to_annotated(query).await; + assert_snapshot!(annotated, @r" + CoalesceBatchesExec: task_count=Maximum(1) + HashJoinExec: task_count=Maximum(1) + CoalescePartitionsExec: task_count=Maximum(1), required_network_boundary=Coalesce + ProjectionExec: task_count=Desired(2) + AggregateExec: task_count=Desired(2) + CoalesceBatchesExec: task_count=Desired(2), required_network_boundary=Shuffle + RepartitionExec: task_count=Desired(3) + AggregateExec: task_count=Desired(3) + CoalesceBatchesExec: task_count=Desired(3) + FilterExec: task_count=Desired(3) + RepartitionExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + ProjectionExec: task_count=Maximum(1) + AggregateExec: task_count=Maximum(1) + CoalesceBatchesExec: task_count=Maximum(1), required_network_boundary=Shuffle + RepartitionExec: task_count=Desired(3) + AggregateExec: task_count=Desired(3) + CoalesceBatchesExec: task_count=Desired(3) + FilterExec: task_count=Desired(3) + RepartitionExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + ") + } + + #[tokio::test] + async fn test_inner_join() { + let query = r#" + SELECT a."MinTemp", b."MaxTemp" FROM weather a INNER JOIN weather b ON a."RainToday" = b."RainToday" + "#; + let annotated = sql_to_annotated(query).await; + assert_snapshot!(annotated, @r" + CoalesceBatchesExec: task_count=Maximum(1) + HashJoinExec: task_count=Maximum(1) + CoalescePartitionsExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + ") + } + + #[tokio::test] + async fn test_distinct() { + let query = r#" + SELECT DISTINCT "RainToday" FROM weather + "#; + let annotated = sql_to_annotated(query).await; + assert_snapshot!(annotated, @r" + AggregateExec: task_count=Desired(2) + CoalesceBatchesExec: task_count=Desired(2), required_network_boundary=Shuffle + RepartitionExec: task_count=Desired(3) + RepartitionExec: task_count=Desired(3) + AggregateExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + ") + } + + #[tokio::test] + async fn test_union_all() { + let query = r#" + SELECT "MinTemp" FROM weather WHERE "RainToday" = 'yes' + UNION ALL + SELECT "MaxTemp" FROM weather WHERE "RainToday" = 'no' + "#; + let annotated = sql_to_annotated(query).await; + assert_snapshot!(annotated, @r" + UnionExec: task_count=Desired(3) + CoalesceBatchesExec: task_count=Desired(3) + FilterExec: task_count=Desired(3) + RepartitionExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + ProjectionExec: task_count=Desired(3) + CoalesceBatchesExec: task_count=Desired(3) + FilterExec: task_count=Desired(3) + RepartitionExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + ") + } + + #[tokio::test] + async fn test_subquery() { + let query = r#" + SELECT * FROM ( + SELECT "MinTemp", "MaxTemp" FROM weather WHERE "RainToday" = 'yes' + ) AS subquery WHERE "MinTemp" > 5 + "#; + let annotated = sql_to_annotated(query).await; + assert_snapshot!(annotated, @r" + CoalesceBatchesExec: task_count=Desired(3) + FilterExec: task_count=Desired(3) + RepartitionExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + ") + } + + #[tokio::test] + async fn test_window_function() { + let query = r#" + SELECT "MinTemp", ROW_NUMBER() OVER (PARTITION BY "RainToday" ORDER BY "MinTemp") as rn + FROM weather + "#; + let annotated = sql_to_annotated(query).await; + assert_snapshot!(annotated, @r" + ProjectionExec: task_count=Desired(3) + BoundedWindowAggExec: task_count=Desired(3) + SortExec: task_count=Desired(3) + CoalesceBatchesExec: task_count=Desired(3), required_network_boundary=Shuffle + RepartitionExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + ") + } + + async fn sql_to_annotated(query: &str) -> String { + let config = SessionConfig::new() + .with_target_partitions(4) + .with_information_schema(true); + + let state = SessionStateBuilder::new() + .with_default_features() + .with_config(config) + .with_distributed_channel_resolver(InMemoryChannelResolver::new(4)) + .build(); + + let ctx = SessionContext::new_with_state(state); + let mut queries = query.split(";").collect_vec(); + let last_query = queries.pop().unwrap(); + + for query in queries { + ctx.sql(query).await.unwrap(); + } + + register_parquet_tables(&ctx).await.unwrap(); + + let df = ctx.sql(last_query).await.unwrap(); + + let annotated = annotate_plan( + df.create_physical_plan().await.unwrap(), + ctx.state_ref().read().config_options().as_ref(), + ) + .expect("failed to annotate plan"); + format!("{annotated:?}") + } +} diff --git a/src/distributed_planner/task_estimator.rs b/src/distributed_planner/task_estimator.rs index 01c7e06e..e4807a39 100644 --- a/src/distributed_planner/task_estimator.rs +++ b/src/distributed_planner/task_estimator.rs @@ -1,93 +1,151 @@ use crate::config_extension_ext::set_distributed_option_extension; -use crate::{ChannelResolver, DistributedConfig, PartitionIsolatorExec}; +use crate::{DistributedConfig, PartitionIsolatorExec}; use datafusion::catalog::memory::DataSourceExec; use datafusion::config::ConfigOptions; use datafusion::datasource::physical_plan::FileScanConfig; -use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; +use datafusion::physical_plan::ExecutionPlan; use datafusion::prelude::SessionConfig; use std::collections::HashSet; use std::fmt::Debug; use std::sync::Arc; +/// Annotation attached to a single [ExecutionPlan] that determines how many distributed tasks +/// it should run on. +#[derive(Debug, Clone)] +pub enum TaskCountAnnotation { + /// The desired number of distributed tasks for this node. The final task count for the + /// annotated node might not be exactly this number, it is more like a hint, so depending + /// on the desired task count of adjacent nodes, the final task count might change. + Desired(usize), + /// Sets a maximum number of distributed tasks for this node. Typically used with the inner + /// value of 1, stating that this node cannot be executed in a distributed fashion. + Maximum(usize), +} + +impl From for usize { + fn from(annotation: TaskCountAnnotation) -> Self { + annotation.as_usize() + } +} + +impl TaskCountAnnotation { + pub fn as_usize(&self) -> usize { + match self { + Self::Desired(desired) => *desired, + Self::Maximum(maximum) => *maximum, + } + } + + pub(crate) fn limit(self, limit: usize) -> Self { + match self { + Self::Desired(desired) => Self::Desired(desired.min(limit)), + Self::Maximum(maximum) => Self::Maximum(maximum.min(limit)), + } + } +} + /// Result of running a [TaskEstimator] on a leaf node. It tells the distributed planner hints -/// about how many tasks should be employed in [Stage]s that contain leaf nodes. +/// about how many tasks should be used in [Stage]s that contain leaf nodes. pub struct TaskEstimation { - /// The amount of tasks that should be used in the [Stage] containing the leaf node. + /// The number of tasks that should be used in the [Stage] containing the leaf node. /// /// Even if implementations get to decide this number, there are situations where it can /// get overridden: /// - If a [Stage] contains multiple leaf nodes, the one that declares the biggest /// task_count wins. - /// - If there are less available workers than this number, the amount of available workers + /// - If there are less available workers than this number, the number of available workers /// is chosen. - pub task_count: usize, - /// If this is set to something, the leaf node will get replaced by this. - /// - /// This can be used by [TaskEstimator] implementations to perform transformations like: - /// - repartitioning the leaf node. - /// - wrapping it with a [PartitionIsolatorExec]. - /// - any other arbitrary modification. - pub new_plan: Option>, + pub task_count: TaskCountAnnotation, } /// Given a leaf node, provides an estimation about how many tasks should be used in the /// stage containing it, and if the leaf node should be replaced by some other. /// /// The distributed planner will try many [TaskEstimator]s in order until one provides an -/// estimation for a specific leaf node. Once that's done, upper stages will get its task -/// count calculated based on wether lower stages are reducing the cardinality of the data +/// estimation for a specific leaf node. Once that's done, upper stages will get their task +/// count calculated based on whether lower stages are reducing the cardinality of the data /// or increasing it. pub trait TaskEstimator { /// Function applied to leaf nodes that returns a [TaskEstimation] hinting how many - /// tasks should be used in the [Stage] containing that leaf node, and if the leaf - /// node itself should be modified somehow. - fn estimate_tasks( + /// tasks should be used in the [Stage] containing that leaf node. + /// + /// All the [TaskEstimator] registered in the session will be applied to the leaf node + /// until one returns an estimation. If no estimation is return from any of the + /// [TaskEstimator]s, then `Maximum(1)` is returned, hinting the distributed planner to not + /// distribute the stage containing that node. + fn tasks_for_leaf_node( &self, plan: &Arc, cfg: &ConfigOptions, ) -> Option; + + /// After a final task_count is decided, taking into account all the leaf nodes in the [Stage], + /// this allows performing a transformation in the leaf nodes for accounting for the fact that + /// they are going to run in multiple tasks. + fn scale_up_leaf_node( + &self, + plan: &Arc, + task_count: usize, + cfg: &ConfigOptions, + ) -> Option>; } impl TaskEstimator for usize { - fn estimate_tasks( + fn tasks_for_leaf_node( &self, _: &Arc, _: &ConfigOptions, ) -> Option { Some(TaskEstimation { - task_count: *self, - new_plan: None, + task_count: TaskCountAnnotation::Desired(*self), }) } + + fn scale_up_leaf_node( + &self, + _: &Arc, + _: usize, + _: &ConfigOptions, + ) -> Option> { + None + } } impl TaskEstimator for Arc { - fn estimate_tasks( + fn tasks_for_leaf_node( &self, plan: &Arc, cfg: &ConfigOptions, ) -> Option { - self.as_ref().estimate_tasks(plan, cfg) + self.as_ref().tasks_for_leaf_node(plan, cfg) + } + + fn scale_up_leaf_node( + &self, + plan: &Arc, + task_count: usize, + cfg: &ConfigOptions, + ) -> Option> { + self.as_ref().scale_up_leaf_node(plan, task_count, cfg) } } impl TaskEstimator for Arc { - fn estimate_tasks( + fn tasks_for_leaf_node( &self, plan: &Arc, cfg: &ConfigOptions, ) -> Option { - self.as_ref().estimate_tasks(plan, cfg) + self.as_ref().tasks_for_leaf_node(plan, cfg) } -} -impl, &ConfigOptions) -> Option> TaskEstimator for F { - fn estimate_tasks( + fn scale_up_leaf_node( &self, plan: &Arc, + task_count: usize, cfg: &ConfigOptions, - ) -> Option { - self(plan, cfg) + ) -> Option> { + self.as_ref().scale_up_leaf_node(plan, task_count, cfg) } } @@ -112,14 +170,14 @@ pub(crate) fn set_distributed_task_estimator( } /// [TaskEstimator] implementation that acts on [DataSourceExec] nodes that contain -/// [FileScanConfig]s data sources (e.g. Parquet or CSV files). it will read the -/// [DistributedConfig].`files_per_task` field and assigns as many task as needed so that +/// [FileScanConfig]s data sources (e.g., Parquet or CSV files). it will read the +/// [DistributedConfig].`files_per_task` field and assigns as many tasks as needed so that /// no task handles more than the configured files. #[derive(Debug)] struct FileScanConfigTaskEstimator; impl TaskEstimator for FileScanConfigTaskEstimator { - fn estimate_tasks( + fn tasks_for_leaf_node( &self, plan: &Arc, cfg: &ConfigOptions, @@ -143,27 +201,37 @@ impl TaskEstimator for FileScanConfigTaskEstimator { // Based on the user-provided files_per_task configuration, do the math to calculate // how many tasks should be used, without surpassing the number of available workers. - let mut task_count = distinct_files.div_ceil(d_cfg.files_per_task); - let workers = match d_cfg.__private_channel_resolver.0.get_urls() { - Ok(urls) => urls.len(), - Err(_) => 1, - }; - task_count = task_count.min(workers); + let task_count = distinct_files.div_ceil(d_cfg.files_per_task); + Some(TaskEstimation { + task_count: TaskCountAnnotation::Desired(task_count), + }) + } + + fn scale_up_leaf_node( + &self, + plan: &Arc, + task_count: usize, + _cfg: &ConfigOptions, + ) -> Option> { + if task_count == 1 { + return Some(Arc::clone(plan)); + } // Based on the task count, attempt to scale up the partitions in the DataSourceExec by // repartitioning it. This will result in a DataSourceExec with potentially a lot of // partitions, but as we are going wrap it with PartitionIsolatorExec that's fine. - let scaled_partitions = task_count * plan.output_partitioning().partition_count(); - let mut plan = Arc::clone(plan); - if let Ok(Some(repartitioned)) = plan.repartitioned(scaled_partitions, cfg) { - plan = repartitioned; - } - plan = Arc::new(PartitionIsolatorExec::new(plan)); + let dse: &DataSourceExec = plan.as_any().downcast_ref()?; + let file_scan: &FileScanConfig = dse.data_source().as_any().downcast_ref()?; - Some(TaskEstimation { - task_count, - new_plan: Some(plan), - }) + let mut new_file_scan = file_scan.clone(); + new_file_scan.file_groups.clear(); + for file_group in file_scan.file_groups.clone() { + new_file_scan + .file_groups + .extend(file_group.split_files(task_count)); + } + let plan = DataSourceExec::from_data_source(new_file_scan); + Some(Arc::new(PartitionIsolatorExec::new(plan, task_count))) } } @@ -176,13 +244,35 @@ pub(crate) struct CombinedTaskEstimator { } impl TaskEstimator for CombinedTaskEstimator { - fn estimate_tasks( + fn tasks_for_leaf_node( &self, plan: &Arc, cfg: &ConfigOptions, ) -> Option { for estimator in &self.user_provided { - if let Some(result) = estimator.estimate_tasks(plan, cfg) { + if let Some(result) = estimator.tasks_for_leaf_node(plan, cfg) { + return Some(result); + } + } + // We want to execute the default estimators last so that the user-provided ones have + // a chance of providing an estimation. + // If none of the user-provided returned an estimation, the default ones are used. + for default_estimator in [&FileScanConfigTaskEstimator as &dyn TaskEstimator] { + if let Some(result) = default_estimator.tasks_for_leaf_node(plan, cfg) { + return Some(result); + } + } + None + } + + fn scale_up_leaf_node( + &self, + plan: &Arc, + task_count: usize, + cfg: &ConfigOptions, + ) -> Option> { + for estimator in &self.user_provided { + if let Some(result) = estimator.scale_up_leaf_node(plan, task_count, cfg) { return Some(result); } } @@ -190,7 +280,7 @@ impl TaskEstimator for CombinedTaskEstimator { // a chance of providing an estimation. // If none of the user-provided returned an estimation, the default ones are used. for default_estimator in [&FileScanConfigTaskEstimator as &dyn TaskEstimator] { - if let Some(result) = default_estimator.estimate_tasks(plan, cfg) { + if let Some(result) = default_estimator.scale_up_leaf_node(plan, task_count, cfg) { return Some(result); } } @@ -239,23 +329,6 @@ mod tests { Ok(()) } - #[tokio::test] - async fn test_file_scan_config_task_estimator_max_workers() -> Result<(), DataFusionError> { - let mut combined = CombinedTaskEstimator::default(); - combined.push(|_: &Arc, _: &ConfigOptions| None); - - let node = make_data_source_exec().await?; - assert_eq!( - combined.task_count(node, |mut cfg| { - cfg.__private_channel_resolver = - ChannelResolverExtension(Arc::new(InMemoryChannelResolver::new(2))); - cfg - }), - 2 - ); - Ok(()) - } - impl CombinedTaskEstimator { fn push(&mut self, value: impl TaskEstimator + Send + Sync + 'static) { self.user_provided.push(Arc::new(value)); @@ -275,7 +348,10 @@ mod tests { ..Default::default() }; cfg.extensions.insert(f(d_cfg)); - self.estimate_tasks(&node, &cfg).unwrap().task_count + self.tasks_for_leaf_node(&node, &cfg) + .unwrap() + .task_count + .as_usize() } } @@ -292,4 +368,23 @@ mod tests { } Ok(plan) } + + impl, &ConfigOptions) -> Option> TaskEstimator for F { + fn tasks_for_leaf_node( + &self, + plan: &Arc, + cfg: &ConfigOptions, + ) -> Option { + self(plan, cfg) + } + + fn scale_up_leaf_node( + &self, + _plan: &Arc, + _task_count: usize, + _cfg: &ConfigOptions, + ) -> Option> { + None + } + } } diff --git a/src/execution_plans/common.rs b/src/execution_plans/common.rs index cf18142a..ffccc375 100644 --- a/src/execution_plans/common.rs +++ b/src/execution_plans/common.rs @@ -1,30 +1,14 @@ use crate::DistributedConfig; use datafusion::arrow::array::RecordBatch; use datafusion::common::runtime::SpawnedTask; -use datafusion::common::{DataFusionError, plan_err}; use datafusion::execution::memory_pool::{MemoryConsumer, MemoryPool}; use datafusion::physical_expr::Partitioning; -use datafusion::physical_plan::{ExecutionPlan, PlanProperties}; +use datafusion::physical_plan::PlanProperties; use futures::{Stream, StreamExt}; use http::HeaderMap; -use std::borrow::Borrow; use std::sync::Arc; use tokio_stream::wrappers::UnboundedReceiverStream; -pub(super) fn require_one_child( - children: L, -) -> Result, DataFusionError> -where - L: AsRef<[T]>, - T: Borrow>, -{ - let children = children.as_ref(); - if children.len() != 1 { - return plan_err!("Expected exactly 1 children, got {}", children.len()); - } - Ok(children[0].borrow().clone()) -} - pub(super) fn scale_partitioning_props( props: &PlanProperties, f: impl FnOnce(usize) -> usize, diff --git a/src/execution_plans/distributed.rs b/src/execution_plans/distributed.rs index e7b2ac44..273beb3a 100644 --- a/src/execution_plans/distributed.rs +++ b/src/execution_plans/distributed.rs @@ -1,6 +1,6 @@ use crate::channel_resolver_ext::get_distributed_channel_resolver; +use crate::common::require_one_child; use crate::distributed_planner::NetworkBoundaryExt; -use crate::execution_plans::common::require_one_child; use crate::protobuf::DistributedCodec; use crate::stage::{ExecutionTask, Stage}; use datafusion::common::exec_err; @@ -62,12 +62,7 @@ impl DistributedExec { let mut rng = rand::thread_rng(); let start_idx = rng.gen_range(0..urls.len()); - let Some(stage) = plan.input_stage() else { - return exec_err!( - "NetworkBoundary '{}' has not been assigned a stage", - plan.name() - ); - }; + let stage = plan.input_stage(); let ready_stage = Stage { query_id: stage.query_id, diff --git a/src/execution_plans/mod.rs b/src/execution_plans/mod.rs index 377c2739..d7719d12 100644 --- a/src/execution_plans/mod.rs +++ b/src/execution_plans/mod.rs @@ -7,6 +7,6 @@ mod partition_isolator; pub use distributed::DistributedExec; pub(crate) use metrics::MetricsWrapperExec; -pub use network_coalesce::{NetworkCoalesceExec, NetworkCoalesceReady}; -pub use network_shuffle::{NetworkShuffleExec, NetworkShuffleReadyExec}; +pub use network_coalesce::NetworkCoalesceExec; +pub use network_shuffle::NetworkShuffleExec; pub use partition_isolator::PartitionIsolatorExec; diff --git a/src/execution_plans/network_coalesce.rs b/src/execution_plans/network_coalesce.rs index 68e60080..9a317051 100644 --- a/src/execution_plans/network_coalesce.rs +++ b/src/execution_plans/network_coalesce.rs @@ -1,16 +1,16 @@ use crate::channel_resolver_ext::get_distributed_channel_resolver; +use crate::common::require_one_child; use crate::config_extension_ext::ContextGrpcMetadata; -use crate::distributed_planner::{InputStageInfo, NetworkBoundary, limit_tasks_err}; +use crate::distributed_planner::NetworkBoundary; use crate::execution_plans::common::{ - manually_propagate_distributed_config, require_one_child, scale_partitioning_props, - spawn_select_all, + manually_propagate_distributed_config, scale_partitioning_props, spawn_select_all, }; use crate::flight_service::DoGet; use crate::metrics::MetricsCollectingStream; use crate::metrics::proto::MetricsSetProto; use crate::protobuf::{StageKey, map_flight_to_datafusion_error, map_status_to_datafusion_error}; use crate::stage::{MaybeEncodedPlan, Stage}; -use crate::{ChannelResolver, DistributedConfig, DistributedTaskContext}; +use crate::{ChannelResolver, DistributedConfig, DistributedTaskContext, ExecutionTask}; use arrow_flight::Ticket; use arrow_flight::decode::FlightRecordBatchStream; use arrow_flight::error::FlightError; @@ -29,6 +29,7 @@ use std::fmt::Formatter; use std::sync::Arc; use tonic::Request; use tonic::metadata::MetadataMap; +use uuid::Uuid; /// [ExecutionPlan] that coalesces partitions from multiple tasks into a single task without /// performing any repartition, and maintaining the same partitioning scheme. @@ -65,27 +66,7 @@ use tonic::metadata::MetadataMap; /// 2. Ready: runs within a distributed stage and queries the next input stage over the network /// using Arrow Flight. #[derive(Debug, Clone)] -pub enum NetworkCoalesceExec { - Pending(NetworkCoalescePending), - Ready(NetworkCoalesceReady), -} - -/// Placeholder version of the [NetworkCoalesceExec] node. It acts as a marker for the -/// distributed optimization step, which will replace it with the appropriate -/// [NetworkCoalesceReady] node. -#[derive(Debug, Clone)] -pub struct NetworkCoalescePending { - properties: PlanProperties, - input_tasks: usize, - input: Arc, -} - -/// Ready version of the [NetworkCoalesceExec] node. This node can be created in -/// just two ways: -/// - by the distributed optimization step based on an original [NetworkCoalescePending] -/// - deserialized from a protobuf plan sent over the network. -#[derive(Debug, Clone)] -pub struct NetworkCoalesceReady { +pub struct NetworkCoalesceExec { /// the properties we advertise for this execution plan pub(crate) properties: PlanProperties, pub(crate) input_stage: Stage, @@ -107,95 +88,51 @@ impl NetworkCoalesceExec { /// partitions into one, for example: /// - [CoalescePartitionsExec] /// - [SortPreservingMergeExec] - pub fn new(input: Arc, input_tasks: usize) -> Self { - Self::Pending(NetworkCoalescePending { - properties: input.properties().clone(), - input_tasks, - input, + pub fn try_new( + input: Arc, + query_id: Uuid, + num: usize, + task_count: usize, + input_task_count: usize, + ) -> Result { + if task_count > 1 { + return plan_err!( + "NetworkCoalesceExec cannot be executed in more than one task, {task_count} where passed." + ); + } + Ok(Self { + properties: scale_partitioning_props(input.properties(), |p| p * input_task_count), + input_stage: Stage { + query_id, + num, + plan: MaybeEncodedPlan::Decoded(input), + tasks: vec![ExecutionTask { url: None }; input_task_count], + }, + metrics_collection: Default::default(), }) } } impl NetworkBoundary for NetworkCoalesceExec { - fn get_input_stage_info(&self, n_tasks: usize) -> Result { - let Self::Pending(pending) = self else { - return plan_err!("can only return wrapped child if on Pending state"); - }; - - // As this node coalesces multiple tasks into 1, it must run in a stage with 1 task. - if n_tasks > 1 { - return Err(limit_tasks_err(1)); - } - - Ok(InputStageInfo { - plan: Arc::clone(&pending.input), - task_count: pending.input_tasks, - }) + fn input_stage(&self) -> &Stage { + &self.input_stage } fn with_input_stage( &self, input_stage: Stage, - ) -> Result, DataFusionError> { - match self { - Self::Pending(pending) => { - let properties = input_stage.plan.decoded()?.properties(); - let ready = NetworkCoalesceReady { - properties: scale_partitioning_props(properties, |p| p * pending.input_tasks), - input_stage, - metrics_collection: Default::default(), - }; - - Ok(Arc::new(Self::Ready(ready))) - } - Self::Ready(ready) => { - let mut ready = ready.clone(); - ready.input_stage = input_stage; - Ok(Arc::new(Self::Ready(ready))) - } - } - } - - fn input_stage(&self) -> Option<&Stage> { - match self { - Self::Pending(_) => None, - Self::Ready(v) => Some(&v.input_stage), - } - } - - fn with_input_task_count( - &self, - input_tasks: usize, - ) -> Result, DataFusionError> { - Ok(Arc::new(match self { - Self::Pending(pending) => Self::Pending(NetworkCoalescePending { - properties: pending.properties.clone(), - input_tasks, - input: pending.input.clone(), - }), - Self::Ready(_) => { - plan_err!("Self can only re-assign input tasks if in 'Pending' state")? - } - })) - } - - fn input_task_count(&self) -> usize { - match self { - Self::Pending(v) => v.input_tasks, - Self::Ready(v) => v.input_stage.tasks.len(), - } + ) -> datafusion::common::Result> { + let mut self_clone = self.clone(); + self_clone.input_stage = input_stage; + Ok(Arc::new(self_clone)) } } impl DisplayAs for NetworkCoalesceExec { fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { - let Self::Ready(self_ready) = self else { - return write!(f, "NetworkCoalesceExec"); - }; - - let input_tasks = self_ready.input_stage.tasks.len(); - let partitions = self_ready.properties.partitioning.partition_count(); - let stage = self_ready.input_stage.num; + let input_tasks = self.input_stage.tasks.len(); + let partitions = self.properties.partitioning.partition_count(); + let stage = self.input_stage.num; write!( f, "[Stage {stage}] => NetworkCoalesceExec: output_partitions={partitions}, input_tasks={input_tasks}", @@ -213,19 +150,13 @@ impl ExecutionPlan for NetworkCoalesceExec { } fn properties(&self) -> &PlanProperties { - match self { - NetworkCoalesceExec::Pending(v) => &v.properties, - NetworkCoalesceExec::Ready(v) => &v.properties, - } + &self.properties } fn children(&self) -> Vec<&Arc> { - match self { - NetworkCoalesceExec::Pending(v) => vec![&v.input], - NetworkCoalesceExec::Ready(v) => match &v.input_stage.plan { - MaybeEncodedPlan::Decoded(v) => vec![v], - MaybeEncodedPlan::Encoded(_) => vec![], - }, + match &self.input_stage.plan { + MaybeEncodedPlan::Decoded(v) => vec![v], + MaybeEncodedPlan::Encoded(_) => vec![], } } @@ -233,18 +164,9 @@ impl ExecutionPlan for NetworkCoalesceExec { self: Arc, children: Vec>, ) -> Result, DataFusionError> { - match self.as_ref() { - Self::Pending(v) => { - let mut v = v.clone(); - v.input = require_one_child(children)?; - Ok(Arc::new(Self::Pending(v))) - } - Self::Ready(v) => { - let mut v = v.clone(); - v.input_stage.plan = MaybeEncodedPlan::Decoded(require_one_child(children)?); - Ok(Arc::new(Self::Ready(v))) - } - } + let mut self_clone = self.as_ref().clone(); + self_clone.input_stage.plan = MaybeEncodedPlan::Decoded(require_one_child(children)?); + Ok(Arc::new(self_clone)) } fn execute( @@ -252,19 +174,13 @@ impl ExecutionPlan for NetworkCoalesceExec { partition: usize, context: Arc, ) -> Result { - let NetworkCoalesceExec::Ready(self_ready) = self else { - return exec_err!( - "NetworkCoalesceExec is not ready, was the distributed optimization step performed?" - ); - }; - // get the channel manager and current stage from our context let channel_resolver = get_distributed_channel_resolver(context.session_config())?; let d_cfg = DistributedConfig::from_config_options(context.session_config().options())?; let retrieve_metrics = d_cfg.collect_metrics; - let input_stage = &self_ready.input_stage; + let input_stage = &self.input_stage; let encoded_input_plan = input_stage.plan.encoded()?; let context_headers = ContextGrpcMetadata::headers_from_ctx(&context); @@ -309,7 +225,7 @@ impl ExecutionPlan for NetworkCoalesceExec { return internal_err!("NetworkCoalesceExec: task is unassigned, cannot proceed"); }; - let metrics_collection_capture = self_ready.metrics_collection.clone(); + let metrics_collection_capture = self.metrics_collection.clone(); let stream = async move { let mut client = channel_resolver.get_flight_client_for_url(&url).await?; let stream = client diff --git a/src/execution_plans/network_shuffle.rs b/src/execution_plans/network_shuffle.rs index b4054043..b2198b88 100644 --- a/src/execution_plans/network_shuffle.rs +++ b/src/execution_plans/network_shuffle.rs @@ -1,7 +1,8 @@ use crate::channel_resolver_ext::get_distributed_channel_resolver; +use crate::common::require_one_child; use crate::config_extension_ext::ContextGrpcMetadata; use crate::execution_plans::common::{ - manually_propagate_distributed_config, require_one_child, scale_partitioning, spawn_select_all, + manually_propagate_distributed_config, scale_partitioning, spawn_select_all, }; use crate::flight_service::DoGet; use crate::metrics::MetricsCollectingStream; @@ -10,7 +11,7 @@ use crate::protobuf::StageKey; use crate::protobuf::{map_flight_to_datafusion_error, map_status_to_datafusion_error}; use crate::stage::{MaybeEncodedPlan, Stage}; use crate::{ - ChannelResolver, DistributedConfig, DistributedTaskContext, InputStageInfo, NetworkBoundary, + ChannelResolver, DistributedConfig, DistributedTaskContext, ExecutionTask, NetworkBoundary, }; use arrow_flight::Ticket; use arrow_flight::decode::FlightRecordBatchStream; @@ -18,7 +19,7 @@ use arrow_flight::error::FlightError; use bytes::Bytes; use dashmap::DashMap; use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; -use datafusion::common::{exec_err, internal_datafusion_err, plan_err}; +use datafusion::common::{internal_datafusion_err, plan_err}; use datafusion::error::DataFusionError; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_expr::Partitioning; @@ -35,6 +36,7 @@ use std::fmt::Formatter; use std::sync::Arc; use tonic::Request; use tonic::metadata::MetadataMap; +use uuid::Uuid; /// [ExecutionPlan] implementation that shuffles data across the network in a distributed context. /// @@ -117,27 +119,7 @@ use tonic::metadata::MetadataMap; /// 2. Ready: runs within a distributed stage and queries the next input stage over the network /// using Arrow Flight. #[derive(Debug, Clone)] -#[allow(clippy::large_enum_variant)] -pub enum NetworkShuffleExec { - Pending(NetworkShufflePendingExec), - Ready(NetworkShuffleReadyExec), -} - -/// Placeholder version of the [NetworkShuffleExec] node. It acts as a marker for the -/// distributed optimization step, which will replace it with the appropriate -/// [NetworkShuffleReadyExec] node. -#[derive(Debug, Clone)] -pub struct NetworkShufflePendingExec { - input: Arc, - input_tasks: usize, -} - -/// Ready version of the [NetworkShuffleExec] node. This node can be created in -/// just two ways: -/// - by the distributed optimization step based on an original [NetworkShufflePendingExec] -/// - deserialized from a protobuf plan sent over the network. -#[derive(Debug, Clone)] -pub struct NetworkShuffleReadyExec { +pub struct NetworkShuffleExec { /// the properties we advertise for this execution plan pub(crate) properties: PlanProperties, pub(crate) input_stage: Stage, @@ -159,31 +141,22 @@ impl NetworkShuffleExec { /// node is a [RepartitionExec] with a [Partitioning::Hash] partition scheme. pub fn try_new( input: Arc, - input_tasks: usize, + query_id: Uuid, + num: usize, + task_count: usize, + input_task_count: usize, ) -> Result { if !matches!(input.output_partitioning(), Partitioning::Hash(_, _)) { return plan_err!("NetworkShuffleExec input must be hash partitioned"); } - Ok(Self::Pending(NetworkShufflePendingExec { - input, - input_tasks, - })) - } -} -impl NetworkBoundary for NetworkShuffleExec { - fn get_input_stage_info(&self, n_tasks: usize) -> Result { - let Self::Pending(pending) = self else { - return plan_err!("cannot only return wrapped child if on Pending state"); - }; - - let transformed = Arc::clone(&pending.input).transform_down(|plan| { + let transformed = Arc::clone(&input).transform_down(|plan| { if let Some(r_exe) = plan.as_any().downcast_ref::() { // Scale the input RepartitionExec to account for all the tasks to which it will // need to fan data out. let scaled = Arc::new(RepartitionExec::try_new( require_one_child(r_exe.children())?, - scale_partitioning(r_exe.partitioning(), |p| p * n_tasks), + scale_partitioning(r_exe.partitioning(), |p| p * task_count), )?); Ok(Transformed::new(scaled, true, TreeNodeRecursion::Stop)) } else if matches!(plan.output_partitioning(), Partitioning::Hash(_, _)) { @@ -198,72 +171,39 @@ impl NetworkBoundary for NetworkShuffleExec { } })?; - Ok(InputStageInfo { - plan: transformed.data, - task_count: pending.input_tasks, + Ok(Self { + input_stage: Stage { + query_id, + num, + plan: MaybeEncodedPlan::Decoded(transformed.data), + tasks: vec![ExecutionTask { url: None }; input_task_count], + }, + properties: input.properties().clone(), + metrics_collection: Default::default(), }) } +} - fn with_input_task_count( - &self, - input_tasks: usize, - ) -> Result, DataFusionError> { - Ok(Arc::new(match self { - Self::Pending(prev) => Self::Pending(NetworkShufflePendingExec { - input: Arc::clone(&prev.input), - input_tasks, - }), - Self::Ready(_) => plan_err!( - "NetworkShuffleExec can only re-assign input tasks if in 'Pending' state" - )?, - })) - } - - fn input_task_count(&self) -> usize { - match self { - Self::Pending(v) => v.input_tasks, - Self::Ready(v) => v.input_stage.tasks.len(), - } +impl NetworkBoundary for NetworkShuffleExec { + fn input_stage(&self) -> &Stage { + &self.input_stage } fn with_input_stage( &self, input_stage: Stage, - ) -> Result, DataFusionError> { - match self { - Self::Pending(pending) => { - let ready = NetworkShuffleReadyExec { - properties: pending.input.properties().clone(), - input_stage, - metrics_collection: Default::default(), - }; - Ok(Arc::new(Self::Ready(ready))) - } - Self::Ready(ready) => { - let mut ready = ready.clone(); - ready.input_stage = input_stage; - Ok(Arc::new(Self::Ready(ready))) - } - } - } - - fn input_stage(&self) -> Option<&Stage> { - match self { - Self::Pending(_) => None, - Self::Ready(v) => Some(&v.input_stage), - } + ) -> datafusion::common::Result> { + let mut self_clone = self.clone(); + self_clone.input_stage = input_stage; + Ok(Arc::new(self_clone)) } } impl DisplayAs for NetworkShuffleExec { fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { - let Self::Ready(self_ready) = self else { - return write!(f, "NetworkShuffleExec: Pending"); - }; - - let input_tasks = self_ready.input_stage.tasks.len(); - let partitions = self_ready.properties.partitioning.partition_count(); - let stage = self_ready.input_stage.num; + let input_tasks = self.input_stage.tasks.len(); + let partitions = self.properties.partitioning.partition_count(); + let stage = self.input_stage.num; write!( f, "[Stage {stage}] => NetworkShuffleExec: output_partitions={partitions}, input_tasks={input_tasks}", @@ -281,19 +221,13 @@ impl ExecutionPlan for NetworkShuffleExec { } fn properties(&self) -> &PlanProperties { - match self { - NetworkShuffleExec::Pending(v) => v.input.properties(), - NetworkShuffleExec::Ready(v) => &v.properties, - } + &self.properties } fn children(&self) -> Vec<&Arc> { - match self { - NetworkShuffleExec::Pending(v) => vec![&v.input], - NetworkShuffleExec::Ready(v) => match &v.input_stage.plan { - MaybeEncodedPlan::Decoded(v) => vec![v], - MaybeEncodedPlan::Encoded(_) => vec![], - }, + match &self.input_stage.plan { + MaybeEncodedPlan::Decoded(v) => vec![v], + MaybeEncodedPlan::Encoded(_) => vec![], } } @@ -301,18 +235,9 @@ impl ExecutionPlan for NetworkShuffleExec { self: Arc, children: Vec>, ) -> Result, DataFusionError> { - match self.as_ref() { - Self::Pending(v) => { - let mut v = v.clone(); - v.input = require_one_child(children)?; - Ok(Arc::new(Self::Pending(v))) - } - Self::Ready(v) => { - let mut v = v.clone(); - v.input_stage.plan = MaybeEncodedPlan::Decoded(require_one_child(children)?); - Ok(Arc::new(Self::Ready(v))) - } - } + let mut self_clone = self.as_ref().clone(); + self_clone.input_stage.plan = MaybeEncodedPlan::Decoded(require_one_child(children)?); + Ok(Arc::new(self_clone)) } fn execute( @@ -320,19 +245,13 @@ impl ExecutionPlan for NetworkShuffleExec { partition: usize, context: Arc, ) -> Result { - let NetworkShuffleExec::Ready(self_ready) = self else { - return exec_err!( - "NetworkShuffleExec is not ready, was the distributed optimization step performed?" - ); - }; - // get the channel manager and current stage from our context let channel_resolver = get_distributed_channel_resolver(context.session_config())?; let d_cfg = DistributedConfig::from_config_options(context.session_config().options())?; let retrieve_metrics = d_cfg.collect_metrics; - let input_stage = &self_ready.input_stage; + let input_stage = &self.input_stage; let encoded_input_plan = input_stage.plan.encoded()?; let input_stage_tasks = input_stage.tasks.to_vec(); @@ -342,7 +261,7 @@ impl ExecutionPlan for NetworkShuffleExec { let context_headers = ContextGrpcMetadata::headers_from_ctx(&context); let task_context = DistributedTaskContext::from_ctx(&context); - let off = self_ready.properties.partitioning.partition_count() * task_context.task_index; + let off = self.properties.partitioning.partition_count() * task_context.task_index; // TODO: this propagation should be automatic https://github.com/datafusion-contrib/datafusion-distributed/issues/247 let context_headers = manually_propagate_distributed_config(context_headers, d_cfg); @@ -365,7 +284,7 @@ impl ExecutionPlan for NetworkShuffleExec { }, ); - let metrics_collection_capture = self_ready.metrics_collection.clone(); + let metrics_collection_capture = self.metrics_collection.clone(); async move { let url = task.url.ok_or(internal_datafusion_err!( "NetworkShuffleExec: task is unassigned, cannot proceed" diff --git a/src/execution_plans/partition_isolator.rs b/src/execution_plans/partition_isolator.rs index 5fb8d088..00f1b51e 100644 --- a/src/execution_plans/partition_isolator.rs +++ b/src/execution_plans/partition_isolator.rs @@ -1,7 +1,5 @@ use crate::DistributedTaskContext; -use crate::distributed_planner::limit_tasks_err; -use datafusion::common::{exec_err, plan_err}; -use datafusion::error::DataFusionError; +use crate::common::require_one_child; use datafusion::execution::TaskContext; use datafusion::physical_plan::ExecutionPlanProperties; use datafusion::{ @@ -50,62 +48,31 @@ use std::{fmt::Formatter, sync::Arc}; /// └───────────────────────────┘ └───────────────────────────┘ └───────────────────────────┘ ■ /// ``` #[derive(Debug)] -#[allow(clippy::large_enum_variant)] -pub enum PartitionIsolatorExec { - Pending(PartitionIsolatorPendingExec), - Ready(PartitionIsolatorReadyExec), -} - -#[derive(Debug)] -pub struct PartitionIsolatorPendingExec { - input: Arc, -} - -#[derive(Debug)] -pub struct PartitionIsolatorReadyExec { +pub struct PartitionIsolatorExec { pub(crate) input: Arc, pub(crate) properties: PlanProperties, pub(crate) n_tasks: usize, } impl PartitionIsolatorExec { - pub fn new(input: Arc) -> Self { - PartitionIsolatorExec::Pending(PartitionIsolatorPendingExec { input }) - } - - pub(crate) fn ready(&self, n_tasks: usize) -> Result { - let Self::Pending(pending) = self else { - return plan_err!("PartitionIsolatorExec is already ready"); - }; - - let input_partitions = pending.input.properties().partitioning.partition_count(); - if n_tasks > input_partitions { - return Err(limit_tasks_err(input_partitions)); - } + pub fn new(input: Arc, n_tasks: usize) -> Self { + let input_partitions = input.properties().partitioning.partition_count(); let partition_count = Self::partition_groups(input_partitions, n_tasks)[0].len(); - let properties = pending - .input + let properties = input .properties() .clone() .with_partitioning(Partitioning::UnknownPartitioning(partition_count)); - Ok(Self::Ready(PartitionIsolatorReadyExec { - input: pending.input.clone(), + Self { + input: input.clone(), properties, n_tasks, - })) - } - - pub(crate) fn new_ready( - input: Arc, - n_tasks: usize, - ) -> Result { - Self::new(input).ready(n_tasks) + } } - pub(crate) fn partition_groups(input_partitions: usize, n_tasks: usize) -> Vec> { + fn partition_groups(input_partitions: usize, n_tasks: usize) -> Vec> { let q = input_partitions / n_tasks; let r = input_partitions % n_tasks; @@ -127,27 +94,16 @@ impl PartitionIsolatorExec { ) -> Vec { Self::partition_groups(input_partitions, n_tasks)[task_i].clone() } - - pub(crate) fn input(&self) -> &Arc { - match self { - PartitionIsolatorExec::Pending(v) => &v.input, - PartitionIsolatorExec::Ready(v) => &v.input, - } - } } impl DisplayAs for PartitionIsolatorExec { fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { - let PartitionIsolatorExec::Ready(self_ready) = self else { - return write!(f, "PartitionIsolatorExec"); - }; - let input_partitions = self.input().output_partitioning().partition_count(); - let partition_groups = - PartitionIsolatorExec::partition_groups(input_partitions, self_ready.n_tasks); + let input_partitions = self.input.output_partitioning().partition_count(); + let partition_groups = Self::partition_groups(input_partitions, self.n_tasks); let n: usize = partition_groups.iter().map(|v| v.len()).sum(); let mut partitions = vec![]; - for _ in 0..self_ready.n_tasks { + for _ in 0..self.n_tasks { partitions.push(vec!["__".to_string(); n]); } @@ -172,33 +128,19 @@ impl ExecutionPlan for PartitionIsolatorExec { } fn properties(&self) -> &PlanProperties { - match self { - PartitionIsolatorExec::Pending(pending) => pending.input.properties(), - PartitionIsolatorExec::Ready(ready) => &ready.properties, - } + &self.properties } fn children(&self) -> Vec<&Arc> { - vec![self.input()] + vec![&self.input] } fn with_new_children( self: Arc, children: Vec>, ) -> Result> { - if children.len() != 1 { - return plan_err!( - "PartitionIsolatorExec wrong number of children, expected 1, got {}", - children.len() - ); - } - - Ok(Arc::new(match self.as_ref() { - PartitionIsolatorExec::Pending(_) => Self::new(children[0].clone()), - PartitionIsolatorExec::Ready(ready) => { - Self::new(children[0].clone()).ready(ready.n_tasks)? - } - })) + let input = require_one_child(children)?; + Ok(Arc::new(Self::new(input, self.n_tasks))) } fn execute( @@ -206,13 +148,9 @@ impl ExecutionPlan for PartitionIsolatorExec { partition: usize, context: Arc, ) -> Result { - let Self::Ready(self_ready) = self else { - return exec_err!("PartitionIsolatorExec is not ready"); - }; - let task_context = DistributedTaskContext::from_ctx(&context); - let input_partitions = self_ready.input.output_partitioning().partition_count(); + let input_partitions = self.input.output_partitioning().partition_count(); let partition_group = Self::partition_group( input_partitions, @@ -228,18 +166,14 @@ impl ExecutionPlan for PartitionIsolatorExec { Some(actual_partition_number) => { if *actual_partition_number >= input_partitions { //trace!("{} returning empty stream", ctx_name); - Ok( - Box::pin(EmptyRecordBatchStream::new(self_ready.input.schema())) - as SendableRecordBatchStream, - ) + Ok(Box::pin(EmptyRecordBatchStream::new(self.input.schema())) + as SendableRecordBatchStream) } else { - self_ready.input.execute(*actual_partition_number, context) + self.input.execute(*actual_partition_number, context) } } - None => Ok( - Box::pin(EmptyRecordBatchStream::new(self_ready.input.schema())) - as SendableRecordBatchStream, - ), + None => Ok(Box::pin(EmptyRecordBatchStream::new(self.input.schema())) + as SendableRecordBatchStream), } } } diff --git a/src/lib.rs b/src/lib.rs index 51896cd7..0f304dd0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,8 +17,8 @@ pub mod test_utils; pub use channel_resolver_ext::{BoxCloneSyncChannel, ChannelResolver, create_flight_client}; pub use distributed_ext::DistributedExt; pub use distributed_planner::{ - DistributedConfig, DistributedPhysicalOptimizerRule, InputStageInfo, NetworkBoundary, - NetworkBoundaryExt, TaskEstimation, TaskEstimator, apply_network_boundaries, distribute_plan, + DistributedConfig, DistributedPhysicalOptimizerRule, NetworkBoundary, NetworkBoundaryExt, + TaskCountAnnotation, TaskEstimation, TaskEstimator, }; pub use execution_plans::{ DistributedExec, NetworkCoalesceExec, NetworkShuffleExec, PartitionIsolatorExec, diff --git a/src/metrics/task_metrics_collector.rs b/src/metrics/task_metrics_collector.rs index 0642a7bb..f8f89495 100644 --- a/src/metrics/task_metrics_collector.rs +++ b/src/metrics/task_metrics_collector.rs @@ -41,19 +41,9 @@ impl TreeNodeRewriter for TaskMetricsCollector { // from child tasks. let metrics_collection = if let Some(node) = plan.as_any().downcast_ref::() { - let NetworkShuffleExec::Ready(ready) = node else { - return internal_err!( - "unexpected NetworkShuffleExec::Pending during metrics collection" - ); - }; - Some(Arc::clone(&ready.metrics_collection)) + Some(Arc::clone(&node.metrics_collection)) } else if let Some(node) = plan.as_any().downcast_ref::() { - let NetworkCoalesceExec::Ready(ready) = node else { - return internal_err!( - "unexpected NetworkCoalesceExec::Pending during metrics collection" - ); - }; - Some(Arc::clone(&ready.metrics_collection)) + Some(Arc::clone(&node.metrics_collection)) } else { None }; @@ -281,18 +271,23 @@ mod tests { let stage = stages.get(&(expected_stage_key.stage_id as usize)).unwrap(); assert_eq!( actual_metrics.len(), - count_plan_nodes(stage.plan.decoded().unwrap()) + count_plan_nodes(stage.plan.decoded().unwrap()), + "Mismatch between collected metrics and actual nodes for {expected_stage_key:?}" ); // Ensure each node has at least one metric which was collected. for metrics_set in actual_metrics.iter() { let metrics_set = metrics_set_proto_to_df(metrics_set).unwrap(); - assert!(metrics_set.iter().count() > 0); + assert!( + metrics_set.iter().count() > 0, + "Did not found metrics for Stage {expected_stage_key:?}" + ); } } } #[tokio::test] + #[ignore] // https://github.com/datafusion-contrib/datafusion-distributed/issues/260 async fn test_metrics_collection_e2e_1() { run_metrics_collection_e2e_test("SELECT id, COUNT(*) as count FROM table1 WHERE id > 1 GROUP BY id ORDER BY id LIMIT 10").await; } @@ -316,6 +311,7 @@ mod tests { } #[tokio::test] + #[ignore] // https://github.com/datafusion-contrib/datafusion-distributed/issues/260 async fn test_metrics_collection_e2e_3() { run_metrics_collection_e2e_test( "SELECT @@ -336,6 +332,7 @@ mod tests { } #[tokio::test] + #[ignore] // https://github.com/datafusion-contrib/datafusion-distributed/issues/260 async fn test_metrics_collection_e2e_4() { run_metrics_collection_e2e_test("SELECT distinct company from table2").await; } diff --git a/src/metrics/task_metrics_rewriter.rs b/src/metrics/task_metrics_rewriter.rs index 4c7d3595..1fe070d9 100644 --- a/src/metrics/task_metrics_rewriter.rs +++ b/src/metrics/task_metrics_rewriter.rs @@ -44,25 +44,18 @@ pub fn rewrite_distributed_plan_with_metrics( let transformed = plan.transform_down(|plan| { // Transform all stages using NetworkShuffleExec and NetworkCoalesceExec as barriers. if let Some(network_boundary) = plan.as_network_boundary() { - return match network_boundary.input_stage() { - Some(stage) => { - // This transform is a bit inefficient because we traverse the plan nodes twice - // For now, we are okay with trading off performance for simplicity. - let plan_with_metrics = - stage_metrics_rewriter(stage, metrics_collection.clone())?; - Ok(Transformed::yes(network_boundary.with_input_stage( - Stage::new( - stage.query_id, - stage.num, - plan_with_metrics, - stage.tasks.len(), - ), - )?)) - } - None => { - internal_err!("Expected input stage to be set in network boundary") - } - }; + let stage = network_boundary.input_stage(); + // This transform is a bit inefficient because we traverse the plan nodes twice + // For now, we are okay with trading off performance for simplicity. + let plan_with_metrics = stage_metrics_rewriter(stage, metrics_collection.clone())?; + return Ok(Transformed::yes(network_boundary.with_input_stage( + Stage::new( + stage.query_id, + stage.num, + plan_with_metrics, + stage.tasks.len(), + ), + )?)); } Ok(Transformed::no(plan)) @@ -473,6 +466,7 @@ mod tests { } #[tokio::test] + #[ignore] // https://github.com/datafusion-contrib/datafusion-distributed/issues/260 async fn test_executed_distributed_plan_has_metrics() { let ctx = make_test_distributed_ctx().await; let plan = ctx diff --git a/src/protobuf/distributed_codec.rs b/src/protobuf/distributed_codec.rs index ee4fb04d..d4c188b0 100644 --- a/src/protobuf/distributed_codec.rs +++ b/src/protobuf/distributed_codec.rs @@ -1,6 +1,6 @@ use super::get_distributed_user_codecs; use crate::NetworkBoundary; -use crate::execution_plans::{NetworkCoalesceExec, NetworkCoalesceReady, NetworkShuffleReadyExec}; +use crate::execution_plans::NetworkCoalesceExec; use crate::stage::{ExecutionTask, MaybeEncodedPlan, Stage}; use crate::{NetworkShuffleExec, PartitionIsolatorExec}; use bytes::Bytes; @@ -146,10 +146,10 @@ impl PhysicalExtensionCodec for DistributedCodec { let child = inputs.first().unwrap(); - Ok(Arc::new(PartitionIsolatorExec::new_ready( + Ok(Arc::new(PartitionIsolatorExec::new( child.clone(), n_tasks as usize, - )?)) + ))) } } } @@ -159,10 +159,7 @@ impl PhysicalExtensionCodec for DistributedCodec { node: Arc, buf: &mut Vec, ) -> datafusion::common::Result<()> { - fn encode_stage_proto(stage: Option<&Stage>) -> Result { - let stage = stage.ok_or(proto_error( - "Cannot encode a NetworkBoundary that has no stage assinged", - ))?; + fn encode_stage_proto(stage: &Stage) -> Result { Ok(StageProto { query_id: Bytes::from(stage.query_id.as_bytes().to_vec()), num: stage.num as u64, @@ -205,13 +202,8 @@ impl PhysicalExtensionCodec for DistributedCodec { wrapper.encode(buf).map_err(|e| proto_error(format!("{e}"))) } else if let Some(node) = node.as_any().downcast_ref::() { - let PartitionIsolatorExec::Ready(ready_node) = node else { - return Err(proto_error( - "deserialized an PartitionIsolatorExec that is not ready", - )); - }; let inner = PartitionIsolatorExecProto { - n_tasks: ready_node.n_tasks as u64, + n_tasks: node.n_tasks as u64, }; let wrapper = DistributedExecProto { @@ -315,7 +307,7 @@ fn new_network_hash_shuffle_exec( schema: SchemaRef, input_stage: Stage, ) -> NetworkShuffleExec { - NetworkShuffleExec::Ready(NetworkShuffleReadyExec { + NetworkShuffleExec { properties: PlanProperties::new( EquivalenceProperties::new(schema), partitioning, @@ -324,7 +316,7 @@ fn new_network_hash_shuffle_exec( ), input_stage, metrics_collection: Default::default(), - }) + } } /// Protobuf representation of the [NetworkShuffleExec] physical node. It serves as @@ -345,7 +337,7 @@ fn new_network_coalesce_tasks_exec( schema: SchemaRef, input_stage: Stage, ) -> NetworkCoalesceExec { - NetworkCoalesceExec::Ready(NetworkCoalesceReady { + NetworkCoalesceExec { properties: PlanProperties::new( EquivalenceProperties::new(schema), partitioning, @@ -354,7 +346,7 @@ fn new_network_coalesce_tasks_exec( ), input_stage, metrics_collection: Default::default(), - }) + } } fn encode_tasks(tasks: &[ExecutionTask]) -> Vec { @@ -451,8 +443,7 @@ mod tests { dummy_stage(), )); - let plan: Arc = - Arc::new(PartitionIsolatorExec::new_ready(flight.clone(), 1)?); + let plan: Arc = Arc::new(PartitionIsolatorExec::new(flight.clone(), 1)); let mut buf = Vec::new(); codec.try_encode(plan.clone(), &mut buf)?; @@ -481,8 +472,7 @@ mod tests { )); let union = UnionExec::try_new(vec![left.clone(), right.clone()])?; - let plan: Arc = - Arc::new(PartitionIsolatorExec::new_ready(union.clone(), 1)?); + let plan: Arc = Arc::new(PartitionIsolatorExec::new(union.clone(), 1)); let mut buf = Vec::new(); codec.try_encode(plan.clone(), &mut buf)?; @@ -514,8 +504,7 @@ mod tests { flight.clone(), )); - let plan: Arc = - Arc::new(PartitionIsolatorExec::new_ready(sort.clone(), 1)?); + let plan: Arc = Arc::new(PartitionIsolatorExec::new(sort.clone(), 1)); let mut buf = Vec::new(); codec.try_encode(plan.clone(), &mut buf)?; @@ -559,8 +548,7 @@ mod tests { dummy_stage(), )); - let plan: Arc = - Arc::new(PartitionIsolatorExec::new_ready(flight.clone(), 1)?); + let plan: Arc = Arc::new(PartitionIsolatorExec::new(flight.clone(), 1)); let mut buf = Vec::new(); codec.try_encode(plan.clone(), &mut buf)?; @@ -589,8 +577,7 @@ mod tests { )); let union = UnionExec::try_new(vec![left.clone(), right.clone()])?; - let plan: Arc = - Arc::new(PartitionIsolatorExec::new_ready(union.clone(), 3)?); + let plan: Arc = Arc::new(PartitionIsolatorExec::new(union.clone(), 3)); let mut buf = Vec::new(); codec.try_encode(plan.clone(), &mut buf)?; diff --git a/src/stage.rs b/src/stage.rs index 6b10db02..b3c5bce4 100644 --- a/src/stage.rs +++ b/src/stage.rs @@ -479,7 +479,7 @@ fn display_plan( node_index += 1; if let Some(node) = plan.as_any().downcast_ref::() { isolator_partition_group = Some(PartitionIsolatorExec::partition_group( - node.input().output_partitioning().partition_count(), + node.input.output_partitioning().partition_count(), task_i, n_tasks, )); @@ -665,7 +665,7 @@ fn display_inter_task_edges( while let Some(plan) = queue.pop_front() { index += 1; if let Some(node) = plan.as_any().downcast_ref::() { - if node.input_stage().is_none_or(|v| v.num != input_stage.num) { + if node.input_stage().num != input_stage.num { continue; } // draw the edges to this node pulling data up from its child @@ -689,7 +689,7 @@ fn display_inter_task_edges( } continue; } else if let Some(node) = plan.as_any().downcast_ref::() { - if node.input_stage().is_none_or(|v| v.num != input_stage.num) { + if node.input_stage().num != input_stage.num { continue; } // draw the edges to this node pulling data up from its child @@ -739,9 +739,7 @@ fn find_input_stages(plan: &dyn ExecutionPlan) -> Vec<&Stage> { let mut result = vec![]; for child in plan.children() { if let Some(plan) = child.as_network_boundary() { - if let Some(stage) = plan.input_stage() { - result.push(stage); - } + result.push(plan.input_stage()); } else { result.extend(find_input_stages(child.as_ref())); } @@ -752,9 +750,7 @@ fn find_input_stages(plan: &dyn ExecutionPlan) -> Vec<&Stage> { pub(crate) fn find_all_stages(plan: &Arc) -> Vec<&Stage> { let mut result = vec![]; if let Some(plan) = plan.as_network_boundary() { - if let Some(stage) = plan.input_stage() { - result.push(stage); - } + result.push(plan.input_stage()); } for child in plan.children() { result.extend(find_all_stages(child)); diff --git a/src/test_utils/plans.rs b/src/test_utils/plans.rs index b7df9105..11c85e4a 100644 --- a/src/test_utils/plans.rs +++ b/src/test_utils/plans.rs @@ -67,9 +67,7 @@ fn find_input_stages(plan: &dyn ExecutionPlan) -> Vec<&Stage> { let mut result = vec![]; for child in plan.children() { if let Some(plan) = child.as_network_boundary() { - if let Some(stage) = plan.input_stage() { - result.push(stage); - } + result.push(plan.input_stage()); } else { result.extend(find_input_stages(child.as_ref())); } diff --git a/tests/introspection.rs b/tests/introspection.rs index 1ec1df70..759c8e8c 100644 --- a/tests/introspection.rs +++ b/tests/introspection.rs @@ -32,12 +32,11 @@ mod tests { assert_snapshot!(physical_distributed_str, @r" - CoalescePartitionsExec - ProjectionExec: expr=[table_catalog@0 as table_catalog, table_schema@1 as table_schema, table_name@2 as table_name, column_name@3 as column_name, data_type@5 as data_type, is_nullable@4 as is_nullable] - CoalesceBatchesExec: target_batch_size=8192 - FilterExec: table_name@2 = weather - RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1 - StreamingTableExec: partition_sizes=1, projection=[table_catalog, table_schema, table_name, column_name, is_nullable, data_type] + ProjectionExec: expr=[table_catalog@0 as table_catalog, table_schema@1 as table_schema, table_name@2 as table_name, column_name@3 as column_name, data_type@5 as data_type, is_nullable@4 as is_nullable] + CoalesceBatchesExec: target_batch_size=8192 + FilterExec: table_name@2 = weather + RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1 + StreamingTableExec: partition_sizes=1, projection=[table_catalog, table_schema, table_name, column_name, is_nullable, data_type] ", ); diff --git a/tests/tpch_explain_analyze.rs b/tests/tpch_explain_analyze.rs index f298156d..fc8f2888 100644 --- a/tests/tpch_explain_analyze.rs +++ b/tests/tpch_explain_analyze.rs @@ -33,14 +33,14 @@ mod tests { │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p11] t1:[p0..p11] t2:[p0..p11] t3:[p0..p11] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=24, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([l_returnflag@0, l_linestatus@1], 12), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[l_returnflag@5 as l_returnflag, l_linestatus@6 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))], metrics=[output_rows=24, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=64, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([l_returnflag@0, l_linestatus@1], 12), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[l_returnflag@5 as l_returnflag, l_linestatus@6 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))], metrics=[output_rows=64, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ ProjectionExec: expr=[l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as __common_expr_1, l_quantity@0 as l_quantity, l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, l_tax@3 as l_tax, l_returnflag@4 as l_returnflag, l_linestatus@5 as l_linestatus], metrics=[output_rows=591856, elapsed_compute=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=591856, elapsed_compute=] │ FilterExec: l_shipdate@6 <= 1998-09-02, projection=[l_quantity@0, l_extendedprice@1, l_discount@2, l_tax@3, l_returnflag@4, l_linestatus@5], metrics=[output_rows=591856, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate], file_type=parquet, predicate=l_shipdate@6 <= 1998-09-02, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@0 <= 1998-09-02, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:..], ...]}, projection=[l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate], file_type=parquet, predicate=l_shipdate@6 <= 1998-09-02, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@0 <= 1998-09-02, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -52,84 +52,72 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], metrics=[output_rows=11264, elapsed_compute=, output_bytes=B] - │ [Stage 7] => NetworkCoalesceExec: output_partitions=24, input_tasks=4, metrics=[] + │ SortExec: expr=[s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=11264, elapsed_compute=, output_bytes=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] + │ ProjectionExec: expr=[s_acctbal@5 as s_acctbal, s_name@2 as s_name, n_name@7 as n_name, p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_address@3 as s_address, s_phone@4 as s_phone, s_comment@6 as s_comment], metrics=[output_rows=11264, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=11264, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@0, ps_partkey@1), (ps_supplycost@7, min(partsupp.ps_supplycost)@0)], projection=[p_partkey@0, p_mfgr@1, s_name@2, s_address@3, s_phone@4, s_acctbal@5, s_comment@6, n_name@8], metrics=[output_rows=11264, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16128, elapsed_compute=, output_bytes=B] + │ RepartitionExec: partitioning=Hash([p_partkey@0, ps_supplycost@7], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16128, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@9)], projection=[p_partkey@1, p_mfgr@2, s_name@3, s_address@4, s_phone@5, s_acctbal@6, s_comment@7, ps_supplycost@8, n_name@9], metrics=[output_rows=16128, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=, output_bytes=B] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] + │ ProjectionExec: expr=[p_partkey@2 as p_partkey, p_mfgr@3 as p_mfgr, s_name@4 as s_name, s_address@5 as s_address, s_phone@6 as s_phone, s_acctbal@7 as s_acctbal, s_comment@8 as s_comment, ps_supplycost@9 as ps_supplycost, n_name@0 as n_name, n_regionkey@1 as n_regionkey], metrics=[output_rows=4672, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=4672, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@4)], projection=[n_name@1, n_regionkey@2, p_partkey@3, p_mfgr@4, s_name@5, s_address@6, s_phone@8, s_acctbal@9, s_comment@10, ps_supplycost@11], metrics=[output_rows=4672, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=, output_bytes=B] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ n_regionkey@2 >= 3 AND n_regionkey@2 <= 3 ], pruning_predicate=n_regionkey_null_count@1 != row_count@2 AND n_regionkey_max@0 >= 3 AND n_regionkey_null_count@1 != row_count@2 AND n_regionkey_min@3 <= 3, required_guarantees=[], metrics=[output_rows=400, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ ProjectionExec: expr=[p_partkey@6 as p_partkey, p_mfgr@7 as p_mfgr, s_name@0 as s_name, s_address@1 as s_address, s_nationkey@2 as s_nationkey, s_phone@3 as s_phone, s_acctbal@4 as s_acctbal, s_comment@5 as s_comment, ps_supplycost@8 as ps_supplycost], metrics=[output_rows=292, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=292, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@2)], projection=[s_name@1, s_address@2, s_nationkey@3, s_phone@4, s_acctbal@5, s_comment@6, p_partkey@7, p_mfgr@8, ps_supplycost@10], metrics=[output_rows=292, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=, output_bytes=B] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment], file_type=parquet, predicate=DynamicFilter [ s_nationkey@3 >= 0 AND s_nationkey@3 <= 24 ], pruning_predicate=s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 0 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 24, required_guarantees=[], metrics=[output_rows=1000, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=292, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, ps_partkey@0)], projection=[p_partkey@0, p_mfgr@1, ps_suppkey@3, ps_supplycost@4], metrics=[output_rows=292, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=73, elapsed_compute=, output_bytes=B] + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ ps_partkey@0 >= 249 AND ps_partkey@0 <= 19455 ] AND DynamicFilter [ ps_suppkey@1 >= 1 AND ps_suppkey@1 <= 1000 ], pruning_predicate=ps_partkey_null_count@1 != row_count@2 AND ps_partkey_max@0 >= 249 AND ps_partkey_null_count@1 != row_count@2 AND ps_partkey_min@3 <= 19455 AND ps_suppkey_null_count@5 != row_count@2 AND ps_suppkey_max@4 >= 1 AND ps_suppkey_null_count@5 != row_count@2 AND ps_suppkey_min@6 <= 1000, required_guarantees=[], metrics=[output_rows=80000, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=11985, elapsed_compute=, output_bytes=B] + │ RepartitionExec: partitioning=Hash([ps_partkey@1, min(partsupp.ps_supplycost)@0], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ ProjectionExec: expr=[min(partsupp.ps_supplycost)@1 as min(partsupp.ps_supplycost), ps_partkey@0 as ps_partkey], metrics=[output_rows=11985, elapsed_compute=, output_bytes=B] + │ AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)], metrics=[output_rows=11985, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=11985, elapsed_compute=, output_bytes=B] + │ RepartitionExec: partitioning=Hash([ps_partkey@0], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)], metrics=[output_rows=11985, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=, reduction_factor=0.29% (11985/4177920)] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=4177920, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@2)], projection=[ps_partkey@1, ps_supplycost@2], metrics=[output_rows=4177920, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=, output_bytes=B] + │ [Stage 3] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] + │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_supplycost@2 as ps_supplycost, n_regionkey@0 as n_regionkey], metrics=[output_rows=1280000, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1280000, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_regionkey@1, ps_partkey@2, ps_supplycost@3], metrics=[output_rows=1280000, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=, output_bytes=B] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ n_regionkey@1 >= 3 AND n_regionkey@1 <= 3 ], pruning_predicate=n_regionkey_null_count@1 != row_count@2 AND n_regionkey_max@0 >= 3 AND n_regionkey_null_count@1 != row_count@2 AND n_regionkey_min@3 <= 3, required_guarantees=[], metrics=[output_rows=400, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_supplycost@2 as ps_supplycost, s_nationkey@0 as s_nationkey], metrics=[output_rows=80000, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=80000, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@1)], projection=[s_nationkey@1, ps_partkey@2, ps_supplycost@4], metrics=[output_rows=80000, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=, output_bytes=B] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ s_nationkey@1 >= 0 AND s_nationkey@1 <= 24 ], pruning_predicate=s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 0 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 24, required_guarantees=[], metrics=[output_rows=1000, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ ps_suppkey@1 >= 1 AND ps_suppkey@1 <= 1000 ], pruning_predicate=ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000, required_guarantees=[], metrics=[output_rows=80000, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ SortExec: expr=[s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=11264, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] - │ ProjectionExec: expr=[s_acctbal@5 as s_acctbal, s_name@2 as s_name, n_name@7 as n_name, p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_address@3 as s_address, s_phone@4 as s_phone, s_comment@6 as s_comment], metrics=[output_rows=11264, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=11264, elapsed_compute=] - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@0, ps_partkey@1), (ps_supplycost@7, min(partsupp.ps_supplycost)@0)], projection=[p_partkey@0, p_mfgr@1, s_name@2, s_address@3, s_phone@4, s_acctbal@5, s_comment@6, n_name@8], metrics=[output_rows=11264, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] - │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=3, metrics=[] + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] + │ FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0], metrics=[output_rows=16, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/region/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/13.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = EUROPE, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= EUROPE AND EUROPE <= r_name_max@1, required_guarantees=[r_name in (EUROPE)], metrics=[output_rows=80, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=73, elapsed_compute=] + │ FilterExec: p_size@3 = 15 AND p_type@2 LIKE %BRASS, projection=[p_partkey@0, p_mfgr@1], metrics=[output_rows=73, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/13.parquet], ...]}, projection=[p_partkey, p_mfgr, p_type, p_size], file_type=parquet, predicate=p_size@3 = 15 AND p_type@2 LIKE %BRASS, pruning_predicate=p_size_null_count@2 != row_count@3 AND p_size_min@0 <= 15 AND 15 <= p_size_max@1, required_guarantees=[p_size in (15)], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] + │ FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0], metrics=[output_rows=16, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/region/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/13.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = EUROPE, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= EUROPE AND EUROPE <= r_name_max@1, required_guarantees=[r_name in (EUROPE)], metrics=[output_rows=80, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p23] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16128, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([p_partkey@0, ps_supplycost@7], 24), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16128, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@9)], projection=[p_partkey@1, p_mfgr@2, s_name@3, s_address@4, s_phone@5, s_acctbal@6, s_comment@7, ps_supplycost@8, n_name@9], metrics=[output_rows=16128, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=] - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ ProjectionExec: expr=[p_partkey@2 as p_partkey, p_mfgr@3 as p_mfgr, s_name@4 as s_name, s_address@5 as s_address, s_phone@6 as s_phone, s_acctbal@7 as s_acctbal, s_comment@8 as s_comment, ps_supplycost@9 as ps_supplycost, n_name@0 as n_name, n_regionkey@1 as n_regionkey], metrics=[output_rows=4672, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=4672, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@4)], projection=[n_name@1, n_regionkey@2, p_partkey@3, p_mfgr@4, s_name@5, s_address@6, s_phone@8, s_acctbal@9, s_comment@10, ps_supplycost@11], metrics=[output_rows=4672, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ ProjectionExec: expr=[p_partkey@6 as p_partkey, p_mfgr@7 as p_mfgr, s_name@0 as s_name, s_address@1 as s_address, s_nationkey@2 as s_nationkey, s_phone@3 as s_phone, s_acctbal@4 as s_acctbal, s_comment@5 as s_comment, ps_supplycost@8 as ps_supplycost], metrics=[output_rows=292, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=292, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@2)], projection=[s_name@1, s_address@2, s_nationkey@3, s_phone@4, s_acctbal@5, s_comment@6, p_partkey@7, p_mfgr@8, ps_supplycost@10], metrics=[output_rows=292, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=292, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, ps_partkey@0)], projection=[p_partkey@0, p_mfgr@1, ps_suppkey@3, ps_supplycost@4], metrics=[output_rows=292, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=73, elapsed_compute=] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], metrics=[output_rows=80000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] - │ FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0], metrics=[output_rows=16, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/region/1.parquet, /testdata/tpch/explain_analyze_sf0.1/region/10.parquet, /testdata/tpch/explain_analyze_sf0.1/region/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/12.parquet, /testdata/tpch/explain_analyze_sf0.1/region/13.parquet, /testdata/tpch/explain_analyze_sf0.1/region/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/15.parquet, /testdata/tpch/explain_analyze_sf0.1/region/16.parquet, /testdata/tpch/explain_analyze_sf0.1/region/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/3.parquet, /testdata/tpch/explain_analyze_sf0.1/region/4.parquet, /testdata/tpch/explain_analyze_sf0.1/region/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/6.parquet, /testdata/tpch/explain_analyze_sf0.1/region/7.parquet, /testdata/tpch/explain_analyze_sf0.1/region/8.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = EUROPE, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= EUROPE AND EUROPE <= r_name_max@1, required_guarantees=[r_name in (EUROPE)], metrics=[output_rows=80, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=73, elapsed_compute=] - │ FilterExec: p_size@3 = 15 AND p_type@2 LIKE %BRASS, projection=[p_partkey@0, p_mfgr@1], metrics=[output_rows=73, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_mfgr, p_type, p_size], file_type=parquet, predicate=p_size@3 = 15 AND p_type@2 LIKE %BRASS, pruning_predicate=p_size_null_count@2 != row_count@3 AND p_size_min@0 <= 15 AND 15 <= p_size_max@1, required_guarantees=[p_size in (15)], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=11985, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([ps_partkey@1, min(partsupp.ps_supplycost)@0], 24), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ ProjectionExec: expr=[min(partsupp.ps_supplycost)@1 as min(partsupp.ps_supplycost), ps_partkey@0 as ps_partkey], metrics=[output_rows=11985, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)], metrics=[output_rows=11985, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] - └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=11985, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([ps_partkey@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)], metrics=[output_rows=11985, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=4177920, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@2)], projection=[ps_partkey@1, ps_supplycost@2], metrics=[output_rows=4177920, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=] - │ [Stage 4] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_supplycost@2 as ps_supplycost, n_regionkey@0 as n_regionkey], metrics=[output_rows=1280000, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1280000, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_regionkey@1, ps_partkey@2, ps_supplycost@3], metrics=[output_rows=1280000, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_supplycost@2 as ps_supplycost, s_nationkey@0 as s_nationkey], metrics=[output_rows=80000, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=80000, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@1)], projection=[s_nationkey@1, ps_partkey@2, ps_supplycost@4], metrics=[output_rows=80000, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=80000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] - │ FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0], metrics=[output_rows=16, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/region/1.parquet, /testdata/tpch/explain_analyze_sf0.1/region/10.parquet, /testdata/tpch/explain_analyze_sf0.1/region/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/12.parquet, /testdata/tpch/explain_analyze_sf0.1/region/13.parquet, /testdata/tpch/explain_analyze_sf0.1/region/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/15.parquet, /testdata/tpch/explain_analyze_sf0.1/region/16.parquet, /testdata/tpch/explain_analyze_sf0.1/region/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/3.parquet, /testdata/tpch/explain_analyze_sf0.1/region/4.parquet, /testdata/tpch/explain_analyze_sf0.1/region/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/6.parquet, /testdata/tpch/explain_analyze_sf0.1/region/7.parquet, /testdata/tpch/explain_analyze_sf0.1/region/8.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = EUROPE, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= EUROPE AND EUROPE <= r_name_max@1, required_guarantees=[r_name in (EUROPE)], metrics=[output_rows=80, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── "); Ok(()) } @@ -140,38 +128,32 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [revenue@1 DESC, o_orderdate@2 ASC NULLS LAST], metrics=[output_rows=1216, elapsed_compute=, output_bytes=B] - │ [Stage 3] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] + │ SortExec: expr=[revenue@1 DESC, o_orderdate@2 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=1216, elapsed_compute=, output_bytes=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] + │ ProjectionExec: expr=[l_orderkey@0 as l_orderkey, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 as revenue, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority], metrics=[output_rows=1216, elapsed_compute=, output_bytes=B] + │ AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=1216, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1216, elapsed_compute=, output_bytes=B] + │ RepartitionExec: partitioning=Hash([l_orderkey@0, o_orderdate@1, o_shippriority@2], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[l_orderkey@2 as l_orderkey, o_orderdate@0 as o_orderdate, o_shippriority@1 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=1216, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=, reduction_factor=37% (1216/3321)] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=3321, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, o_shippriority@2, l_orderkey@3, l_extendedprice@4, l_discount@5], metrics=[output_rows=3321, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=15224, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=15224, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_custkey@0, o_custkey@1)], projection=[o_orderkey@1, o_orderdate@3, o_shippriority@4], metrics=[output_rows=15224, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=3111, elapsed_compute=, output_bytes=B] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=72678, elapsed_compute=, output_bytes=B] + │ FilterExec: o_orderdate@2 < 1995-03-15, metrics=[output_rows=72678, elapsed_compute=, output_bytes=B, selectivity=48% (72678/150000)] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_orderdate, o_shippriority], file_type=parquet, predicate=o_orderdate@2 < 1995-03-15 AND DynamicFilter [ o_custkey@1 >= 1 AND o_custkey@1 <= 14984 ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@0 < 1995-03-15 AND o_custkey_null_count@4 != row_count@2 AND o_custkey_max@3 >= 1 AND o_custkey_null_count@4 != row_count@2 AND o_custkey_min@5 <= 14984, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=324322, elapsed_compute=, output_bytes=B] + │ FilterExec: l_shipdate@3 > 1995-03-15, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2], metrics=[output_rows=324322, elapsed_compute=, output_bytes=B, selectivity=54% (324322/600572)] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 > 1995-03-15 AND DynamicFilter [ l_orderkey@0 >= 5 AND l_orderkey@0 <= 599846 ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 > 1995-03-15 AND l_orderkey_null_count@4 != row_count@2 AND l_orderkey_max@3 >= 5 AND l_orderkey_null_count@4 != row_count@2 AND l_orderkey_min@5 <= 599846, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ SortExec: expr=[revenue@1 DESC, o_orderdate@2 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=1216, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] - │ ProjectionExec: expr=[l_orderkey@0 as l_orderkey, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 as revenue, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority], metrics=[output_rows=1216, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=1216, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=3111, elapsed_compute=] + │ FilterExec: c_mktsegment@1 = BUILDING, projection=[c_custkey@0], metrics=[output_rows=3111, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/13.parquet], ...]}, projection=[c_custkey, c_mktsegment], file_type=parquet, predicate=c_mktsegment@1 = BUILDING, pruning_predicate=c_mktsegment_null_count@2 != row_count@3 AND c_mktsegment_min@0 <= BUILDING AND BUILDING <= c_mktsegment_max@1, required_guarantees=[c_mktsegment in (BUILDING)], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1216, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([l_orderkey@0, o_orderdate@1, o_shippriority@2], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[l_orderkey@2 as l_orderkey, o_orderdate@0 as o_orderdate, o_shippriority@1 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=1216, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=3321, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, o_shippriority@2, l_orderkey@3, l_extendedprice@4, l_discount@5], metrics=[output_rows=3321, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=15224, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=15224, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_custkey@0, o_custkey@1)], projection=[o_orderkey@1, o_orderdate@3, o_shippriority@4], metrics=[output_rows=15224, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=3111, elapsed_compute=] - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=72678, elapsed_compute=] - │ FilterExec: o_orderdate@2 < 1995-03-15, metrics=[output_rows=72678, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_orderdate, o_shippriority], file_type=parquet, predicate=o_orderdate@2 < 1995-03-15 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@0 < 1995-03-15, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=324322, elapsed_compute=] - │ FilterExec: l_shipdate@3 > 1995-03-15, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2], metrics=[output_rows=324322, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 > 1995-03-15 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 > 1995-03-15, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=3111, elapsed_compute=] - │ FilterExec: c_mktsegment@1 = BUILDING, projection=[c_custkey@0], metrics=[output_rows=3111, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_mktsegment], file_type=parquet, predicate=c_mktsegment@1 = BUILDING, pruning_predicate=c_mktsegment_null_count@2 != row_count@3 AND c_mktsegment_min@0 <= BUILDING AND BUILDING <= c_mktsegment_max@1, required_guarantees=[c_mktsegment in (BUILDING)], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── "); Ok(()) } @@ -181,33 +163,27 @@ mod tests { let plan = test_tpch_query(4).await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [o_orderpriority@0 ASC NULLS LAST], metrics=[output_rows=5, elapsed_compute=, output_bytes=] - │ [Stage 3] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] + │ SortPreservingMergeExec: [o_orderpriority@0 ASC NULLS LAST], metrics=[output_rows=5, elapsed_compute=, output_bytes=B] + │ SortExec: expr=[o_orderpriority@0 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=5, elapsed_compute=, output_bytes=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] + │ ProjectionExec: expr=[o_orderpriority@0 as o_orderpriority, count(Int64(1))@1 as order_count], metrics=[output_rows=5, elapsed_compute=, output_bytes=B] + │ AggregateExec: mode=FinalPartitioned, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))], metrics=[output_rows=5, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=30, elapsed_compute=, output_bytes=B] + │ RepartitionExec: partitioning=Hash([o_orderpriority@0], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))], metrics=[output_rows=30, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=, reduction_factor=0.59% (30/5093)] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5093, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(l_orderkey@0, o_orderkey@0)], projection=[o_orderpriority@1], metrics=[output_rows=5093, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=379809, elapsed_compute=, output_bytes=B] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=24, input_tasks=4, metrics=[] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5552, elapsed_compute=, output_bytes=B] + │ FilterExec: o_orderdate@1 >= 1993-07-01 AND o_orderdate@1 < 1993-10-01, projection=[o_orderkey@0, o_orderpriority@2], metrics=[output_rows=5552, elapsed_compute=, output_bytes=B, selectivity=3.7% (5552/150000)] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_orderdate, o_orderpriority], file_type=parquet, predicate=o_orderdate@1 >= 1993-07-01 AND o_orderdate@1 < 1993-10-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-07-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1993-10-01, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ SortExec: expr=[o_orderpriority@0 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=5, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] - │ ProjectionExec: expr=[o_orderpriority@0 as o_orderpriority, count(Int64(1))@1 as order_count], metrics=[output_rows=5, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))], metrics=[output_rows=5, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] t3:[p18..p23] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=379809, elapsed_compute=] + │ FilterExec: l_receiptdate@2 > l_commitdate@1, projection=[l_orderkey@0], metrics=[output_rows=379809, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:..], ...]}, projection=[l_orderkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@2 > l_commitdate@1, metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=30, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([o_orderpriority@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))], metrics=[output_rows=30, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5093, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(l_orderkey@0, o_orderkey@0)], projection=[o_orderpriority@1], metrics=[output_rows=5093, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=379809, elapsed_compute=] - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5552, elapsed_compute=] - │ FilterExec: o_orderdate@1 >= 1993-07-01 AND o_orderdate@1 < 1993-10-01, projection=[o_orderkey@0, o_orderpriority@2], metrics=[output_rows=5552, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_orderdate, o_orderpriority], file_type=parquet, predicate=o_orderdate@1 >= 1993-07-01 AND o_orderdate@1 < 1993-10-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-07-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1993-10-01, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=379809, elapsed_compute=] - │ FilterExec: l_receiptdate@2 > l_commitdate@1, projection=[l_orderkey@0], metrics=[output_rows=379809, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@2 > l_commitdate@1, metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── "); Ok(()) } @@ -218,55 +194,49 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [revenue@1 DESC], metrics=[output_rows=5, elapsed_compute=, output_bytes=] - │ [Stage 4] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] + │ SortExec: expr=[revenue@1 DESC], preserve_partitioning=[true], metrics=[output_rows=5, elapsed_compute=, output_bytes=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] + │ ProjectionExec: expr=[n_name@0 as n_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as revenue], metrics=[output_rows=5, elapsed_compute=, output_bytes=B] + │ AggregateExec: mode=FinalPartitioned, gby=[n_name@0 as n_name], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=5, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=30, elapsed_compute=, output_bytes=B] + │ RepartitionExec: partitioning=Hash([n_name@0], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[n_name@2 as n_name], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=30, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=, reduction_factor=0.014% (30/221440)] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=221440, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@3)], projection=[l_extendedprice@1, l_discount@2, n_name@3], metrics=[output_rows=221440, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=, output_bytes=B] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] + │ ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, n_name@0 as n_name, n_regionkey@1 as n_regionkey], metrics=[output_rows=59040, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=59040, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_name@1, n_regionkey@2, l_extendedprice@3, l_discount@4], metrics=[output_rows=59040, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=, output_bytes=B] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ n_regionkey@2 >= 2 AND n_regionkey@2 <= 2 ], pruning_predicate=n_regionkey_null_count@1 != row_count@2 AND n_regionkey_max@0 >= 2 AND n_regionkey_null_count@1 != row_count@2 AND n_regionkey_min@3 <= 2, required_guarantees=[], metrics=[output_rows=400, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, s_nationkey@0 as s_nationkey], metrics=[output_rows=3690, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=3690, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1), (s_nationkey@1, c_nationkey@0)], projection=[s_nationkey@1, l_extendedprice@4, l_discount@5], metrics=[output_rows=3690, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=, output_bytes=B] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ s_nationkey@1 >= 0 AND s_nationkey@1 <= 24 ], pruning_predicate=s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 0 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 24, required_guarantees=[], metrics=[output_rows=1000, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=92293, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@1, l_orderkey@0)], projection=[c_nationkey@0, l_suppkey@3, l_extendedprice@4, l_discount@5], metrics=[output_rows=92293, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=22958, elapsed_compute=, output_bytes=B] + │ ProjectionExec: expr=[c_nationkey@1 as c_nationkey, o_orderkey@0 as o_orderkey], metrics=[output_rows=22958, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=22958, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_custkey@1, c_custkey@0)], projection=[o_orderkey@0, c_nationkey@3], metrics=[output_rows=22958, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=22958, elapsed_compute=, output_bytes=B] + │ [Stage 2] => NetworkCoalesceExec: output_partitions=24, input_tasks=4, metrics=[] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ c_custkey@0 >= 1 AND c_custkey@0 <= 14999 ], pruning_predicate=c_custkey_null_count@1 != row_count@2 AND c_custkey_max@0 >= 1 AND c_custkey_null_count@1 != row_count@2 AND c_custkey_min@3 <= 14999, required_guarantees=[], metrics=[output_rows=15000, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ l_orderkey@0 >= 5 AND l_orderkey@0 <= 599943 ], pruning_predicate=l_orderkey_null_count@1 != row_count@2 AND l_orderkey_max@0 >= 5 AND l_orderkey_null_count@1 != row_count@2 AND l_orderkey_min@3 <= 599943, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ SortExec: expr=[revenue@1 DESC], preserve_partitioning=[true], metrics=[output_rows=5, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] - │ ProjectionExec: expr=[n_name@0 as n_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as revenue], metrics=[output_rows=5, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[n_name@0 as n_name], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=5, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] + │ FilterExec: r_name@1 = ASIA, projection=[r_regionkey@0], metrics=[output_rows=16, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/region/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/13.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = ASIA, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= ASIA AND ASIA <= r_name_max@1, required_guarantees=[r_name in (ASIA)], metrics=[output_rows=80, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] t3:[p18..p23] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=22958, elapsed_compute=] + │ FilterExec: o_orderdate@2 >= 1994-01-01 AND o_orderdate@2 < 1995-01-01, projection=[o_orderkey@0, o_custkey@1], metrics=[output_rows=22958, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1994-01-01 AND o_orderdate@2 < 1995-01-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1994-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1995-01-01, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=30, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([n_name@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[n_name@2 as n_name], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=30, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=221440, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@3)], projection=[l_extendedprice@1, l_discount@2, n_name@3], metrics=[output_rows=221440, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=] - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, n_name@0 as n_name, n_regionkey@1 as n_regionkey], metrics=[output_rows=59040, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=59040, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_name@1, n_regionkey@2, l_extendedprice@3, l_discount@4], metrics=[output_rows=59040, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, s_nationkey@0 as s_nationkey], metrics=[output_rows=3690, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=3690, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1), (s_nationkey@1, c_nationkey@0)], projection=[s_nationkey@1, l_extendedprice@4, l_discount@5], metrics=[output_rows=3690, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=92293, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@1, l_orderkey@0)], projection=[c_nationkey@0, l_suppkey@3, l_extendedprice@4, l_discount@5], metrics=[output_rows=92293, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=22958, elapsed_compute=] - │ ProjectionExec: expr=[c_nationkey@1 as c_nationkey, o_orderkey@0 as o_orderkey], metrics=[output_rows=22958, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=22958, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_custkey@1, c_custkey@0)], projection=[o_orderkey@0, c_nationkey@3], metrics=[output_rows=22958, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=22958, elapsed_compute=] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] - │ FilterExec: r_name@1 = ASIA, projection=[r_regionkey@0], metrics=[output_rows=16, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/region/1.parquet, /testdata/tpch/explain_analyze_sf0.1/region/10.parquet, /testdata/tpch/explain_analyze_sf0.1/region/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/12.parquet, /testdata/tpch/explain_analyze_sf0.1/region/13.parquet, /testdata/tpch/explain_analyze_sf0.1/region/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/15.parquet, /testdata/tpch/explain_analyze_sf0.1/region/16.parquet, /testdata/tpch/explain_analyze_sf0.1/region/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/3.parquet, /testdata/tpch/explain_analyze_sf0.1/region/4.parquet, /testdata/tpch/explain_analyze_sf0.1/region/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/6.parquet, /testdata/tpch/explain_analyze_sf0.1/region/7.parquet, /testdata/tpch/explain_analyze_sf0.1/region/8.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = ASIA, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= ASIA AND ASIA <= r_name_max@1, required_guarantees=[r_name in (ASIA)], metrics=[output_rows=80, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=22958, elapsed_compute=] - │ FilterExec: o_orderdate@2 >= 1994-01-01 AND o_orderdate@2 < 1995-01-01, projection=[o_orderkey@0, o_custkey@1], metrics=[output_rows=22958, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1994-01-01 AND o_orderdate@2 < 1995-01-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1994-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1995-01-01, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── "); Ok(()) } @@ -278,15 +248,15 @@ mod tests { ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[sum(lineitem.l_extendedprice * lineitem.l_discount)@0 as revenue], metrics=[output_rows=1, elapsed_compute=, output_bytes=] │ AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice * lineitem.l_discount)], metrics=[output_rows=1, elapsed_compute=, output_bytes=] - │ CoalescePartitionsExec, metrics=[output_rows=8, elapsed_compute=, output_bytes=] - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] + │ CoalescePartitionsExec, metrics=[output_rows=24, elapsed_compute=, output_bytes=B] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=24, input_tasks=4, metrics=[] └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice * lineitem.l_discount)], metrics=[output_rows=8, elapsed_compute=] + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] t3:[p18..p23] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice * lineitem.l_discount)], metrics=[output_rows=24, elapsed_compute=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=11618, elapsed_compute=] │ FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND l_discount@2 >= Some(5),15,2 AND l_discount@2 <= Some(7),15,2 AND l_quantity@0 < Some(2400),15,2, projection=[l_extendedprice@1, l_discount@2], metrics=[output_rows=11618, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_quantity, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND l_discount@2 >= Some(5),15,2 AND l_discount@2 <= Some(7),15,2 AND l_quantity@0 < Some(2400),15,2, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01 AND l_discount_null_count@5 != row_count@2 AND l_discount_max@4 >= Some(5),15,2 AND l_discount_null_count@5 != row_count@2 AND l_discount_min@6 <= Some(7),15,2 AND l_quantity_null_count@8 != row_count@2 AND l_quantity_min@7 < Some(2400),15,2, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:..], ...]}, projection=[l_quantity, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND l_discount@2 >= Some(5),15,2 AND l_discount@2 <= Some(7),15,2 AND l_quantity@0 < Some(2400),15,2, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01 AND l_discount_null_count@5 != row_count@2 AND l_discount_max@4 >= Some(5),15,2 AND l_discount_null_count@5 != row_count@2 AND l_discount_min@6 <= Some(7),15,2 AND l_quantity_null_count@8 != row_count@2 AND l_quantity_min@7 < Some(2400),15,2, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -298,77 +268,71 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST], metrics=[output_rows=4, elapsed_compute=, output_bytes=] - │ [Stage 8] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] + │ SortExec: expr=[supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=4, elapsed_compute=, output_bytes=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] + │ ProjectionExec: expr=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year, sum(shipping.volume)@3 as revenue], metrics=[output_rows=4, elapsed_compute=, output_bytes=] + │ AggregateExec: mode=FinalPartitioned, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)], metrics=[output_rows=4, elapsed_compute=, output_bytes=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=24, elapsed_compute=, output_bytes=B] + │ RepartitionExec: partitioning=Hash([supp_nation@0, cust_nation@1, l_year@2], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)], metrics=[output_rows=24, elapsed_compute=, output_bytes=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=, reduction_factor=0.015% (24/164608)] + │ ProjectionExec: expr=[n_name@4 as supp_nation, n_name@0 as cust_nation, date_part(YEAR, l_shipdate@3) as l_year, l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as volume], metrics=[output_rows=164608, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=164608, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@3)], filter=n_name@0 = FRANCE AND n_name@1 = GERMANY OR n_name@0 = GERMANY AND n_name@1 = FRANCE, projection=[n_name@1, l_extendedprice@2, l_discount@3, l_shipdate@4, n_name@6], metrics=[output_rows=164608, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=32, elapsed_compute=, output_bytes=B] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] + │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, l_shipdate@3 as l_shipdate, c_nationkey@4 as c_nationkey, n_name@0 as n_name], metrics=[output_rows=250096, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=250096, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@0)], projection=[n_name@1, l_extendedprice@3, l_discount@4, l_shipdate@5, c_nationkey@6], metrics=[output_rows=250096, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=32, elapsed_compute=, output_bytes=B] + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] + │ ProjectionExec: expr=[s_nationkey@1 as s_nationkey, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, l_shipdate@4 as l_shipdate, c_nationkey@0 as c_nationkey], metrics=[output_rows=182762, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=182762, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c_custkey@0, o_custkey@4)], projection=[c_nationkey@1, s_nationkey@2, l_extendedprice@3, l_discount@4, l_shipdate@5], metrics=[output_rows=182762, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] + │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ SortExec: expr=[supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=4, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] - │ ProjectionExec: expr=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year, sum(shipping.volume)@3 as revenue], metrics=[output_rows=4, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)], metrics=[output_rows=4, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ [Stage 7] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32, elapsed_compute=] + │ FilterExec: n_name@1 = GERMANY OR n_name@1 = FRANCE, metrics=[output_rows=32, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY OR n_name@1 = FRANCE, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1 OR n_name_null_count@2 != row_count@3 AND n_name_min@0 <= FRANCE AND FRANCE <= n_name_max@1, required_guarantees=[n_name in (FRANCE, GERMANY)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=24, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([supp_nation@0, cust_nation@1, l_year@2], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)], metrics=[output_rows=24, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ ProjectionExec: expr=[n_name@4 as supp_nation, n_name@0 as cust_nation, date_part(YEAR, l_shipdate@3) as l_year, l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as volume], metrics=[output_rows=164608, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=164608, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@3)], filter=n_name@0 = FRANCE AND n_name@1 = GERMANY OR n_name@0 = GERMANY AND n_name@1 = FRANCE, projection=[n_name@1, l_extendedprice@2, l_discount@3, l_shipdate@4, n_name@6], metrics=[output_rows=164608, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=32, elapsed_compute=] - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, l_shipdate@3 as l_shipdate, c_nationkey@4 as c_nationkey, n_name@0 as n_name], metrics=[output_rows=250096, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=250096, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@0)], projection=[n_name@1, l_extendedprice@3, l_discount@4, l_shipdate@5, c_nationkey@6], metrics=[output_rows=250096, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=32, elapsed_compute=] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ ProjectionExec: expr=[s_nationkey@1 as s_nationkey, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, l_shipdate@4 as l_shipdate, c_nationkey@0 as c_nationkey], metrics=[output_rows=182762, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=182762, elapsed_compute=] - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c_custkey@0, o_custkey@4)], projection=[c_nationkey@1, s_nationkey@2, l_extendedprice@3, l_discount@4, l_shipdate@5], metrics=[output_rows=182762, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] - │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32, elapsed_compute=] + │ FilterExec: n_name@1 = FRANCE OR n_name@1 = GERMANY, metrics=[output_rows=32, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = FRANCE OR n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= FRANCE AND FRANCE <= n_name_max@1 OR n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (FRANCE, GERMANY)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=15000, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([c_custkey@0], 6), input_partitions=4, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/13.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ c_nationkey@1 >= 6 AND c_nationkey@1 <= 7 ], pruning_predicate=c_nationkey_null_count@1 != row_count@2 AND c_nationkey_max@0 >= 6 AND c_nationkey_null_count@1 != row_count@2 AND c_nationkey_min@3 <= 7, required_guarantees=[], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=182762, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([o_custkey@4], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=182762, elapsed_compute=] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_orderkey@1, o_orderkey@0)], projection=[s_nationkey@0, l_extendedprice@2, l_discount@3, l_shipdate@4, o_custkey@6], metrics=[output_rows=182762, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] + │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p23] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=182762, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([l_orderkey@1], 24), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=182762, elapsed_compute=] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_nationkey@1, l_orderkey@2, l_extendedprice@4, l_discount@5, l_shipdate@6], metrics=[output_rows=182762, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ s_nationkey@1 >= 6 AND s_nationkey@1 <= 7 ], pruning_predicate=s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 6 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 7, required_guarantees=[], metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=182762, elapsed_compute=] + │ FilterExec: l_shipdate@4 >= 1995-01-01 AND l_shipdate@4 <= 1996-12-31, metrics=[output_rows=182762, elapsed_compute=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@4 >= 1995-01-01 AND l_shipdate@4 <= 1996-12-31 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 <= 1996-12-31, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=150000, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([o_orderkey@0], 24), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:..], ...]}, projection=[o_orderkey, o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ o_custkey@1 >= 32 AND o_custkey@1 <= 14994 OR o_custkey@1 >= 9 AND o_custkey@1 <= 15000 OR o_custkey@1 >= 5 AND o_custkey@1 <= 14997 OR o_custkey@1 >= 8 AND o_custkey@1 <= 14998 OR o_custkey@1 >= 3 AND o_custkey@1 <= 14993 OR o_custkey@1 >= 1 AND o_custkey@1 <= 14999 ], pruning_predicate=o_custkey_null_count@1 != row_count@2 AND o_custkey_max@0 >= 32 AND o_custkey_null_count@1 != row_count@2 AND o_custkey_min@3 <= 14994 OR o_custkey_null_count@1 != row_count@2 AND o_custkey_max@0 >= 9 AND o_custkey_null_count@1 != row_count@2 AND o_custkey_min@3 <= 15000 OR o_custkey_null_count@1 != row_count@2 AND o_custkey_max@0 >= 5 AND o_custkey_null_count@1 != row_count@2 AND o_custkey_min@3 <= 14997 OR o_custkey_null_count@1 != row_count@2 AND o_custkey_max@0 >= 8 AND o_custkey_null_count@1 != row_count@2 AND o_custkey_min@3 <= 14998 OR o_custkey_null_count@1 != row_count@2 AND o_custkey_max@0 >= 3 AND o_custkey_null_count@1 != row_count@2 AND o_custkey_min@3 <= 14993 OR o_custkey_null_count@1 != row_count@2 AND o_custkey_max@0 >= 1 AND o_custkey_null_count@1 != row_count@2 AND o_custkey_min@3 <= 14999, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32, elapsed_compute=] - │ FilterExec: n_name@1 = GERMANY OR n_name@1 = FRANCE, metrics=[output_rows=32, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY OR n_name@1 = FRANCE, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1 OR n_name_null_count@2 != row_count@3 AND n_name_min@0 <= FRANCE AND FRANCE <= n_name_max@1, required_guarantees=[n_name in (FRANCE, GERMANY)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32, elapsed_compute=] - │ FilterExec: n_name@1 = FRANCE OR n_name@1 = GERMANY, metrics=[output_rows=32, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = FRANCE OR n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= FRANCE AND FRANCE <= n_name_max@1 OR n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (FRANCE, GERMANY)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=15000, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([c_custkey@0], 6), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=182762, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([o_custkey@4], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=182762, elapsed_compute=] - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_orderkey@1, o_orderkey@0)], projection=[s_nationkey@0, l_extendedprice@2, l_discount@3, l_shipdate@4, o_custkey@6], metrics=[output_rows=182762, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] - │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p23] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=182762, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([l_orderkey@1], 24), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=182762, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_nationkey@1, l_orderkey@2, l_extendedprice@4, l_discount@5, l_shipdate@6], metrics=[output_rows=182762, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=182762, elapsed_compute=] - │ FilterExec: l_shipdate@4 >= 1995-01-01 AND l_shipdate@4 <= 1996-12-31, metrics=[output_rows=182762, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@4 >= 1995-01-01 AND l_shipdate@4 <= 1996-12-31 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 <= 1996-12-31, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=150000, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([o_orderkey@0], 24), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── "); Ok(()) } @@ -379,87 +343,81 @@ mod tests { assert_snapshot!(plan, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [o_year@0 ASC NULLS LAST], metrics=[output_rows=2, elapsed_compute=, output_bytes=] - │ [Stage 8] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] + │ SortExec: expr=[o_year@0 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=2, elapsed_compute=, output_bytes=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] + │ ProjectionExec: expr=[o_year@0 as o_year, sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END)@1 / sum(all_nations.volume)@2 as mkt_share], metrics=[output_rows=2, elapsed_compute=, output_bytes=] + │ AggregateExec: mode=FinalPartitioned, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)], metrics=[output_rows=2, elapsed_compute=, output_bytes=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=12, elapsed_compute=, output_bytes=B] + │ RepartitionExec: partitioning=Hash([o_year@0], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)], metrics=[output_rows=12, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=, reduction_factor=0.001% (12/1155072)] + │ ProjectionExec: expr=[date_part(YEAR, o_orderdate@2) as o_year, l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as volume, n_name@3 as nation], metrics=[output_rows=1155072, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1155072, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@3)], projection=[l_extendedprice@1, l_discount@2, o_orderdate@3, n_name@5], metrics=[output_rows=1155072, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=, output_bytes=B] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] + │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, o_orderdate@3 as o_orderdate, n_regionkey@4 as n_regionkey, n_name@0 as n_name], metrics=[output_rows=365824, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=365824, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_name@1, l_extendedprice@2, l_discount@3, o_orderdate@5, n_regionkey@6], metrics=[output_rows=365824, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=, output_bytes=B] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, metrics=[output_rows=400, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, s_nationkey@3 as s_nationkey, o_orderdate@4 as o_orderdate, n_regionkey@0 as n_regionkey], metrics=[output_rows=22864, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=22864, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@4)], projection=[n_regionkey@1, l_extendedprice@2, l_discount@3, s_nationkey@4, o_orderdate@5], metrics=[output_rows=22864, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=, output_bytes=B] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ n_regionkey@1 >= 1 AND n_regionkey@1 <= 1 ], pruning_predicate=n_regionkey_null_count@1 != row_count@2 AND n_regionkey_max@0 >= 1 AND n_regionkey_null_count@1 != row_count@2 AND n_regionkey_min@3 <= 1, required_guarantees=[], metrics=[output_rows=400, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1429, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_custkey@3, c_custkey@0)], projection=[l_extendedprice@0, l_discount@1, s_nationkey@2, o_orderdate@4, c_nationkey@6], metrics=[output_rows=1429, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=3, metrics=[] + │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ SortExec: expr=[o_year@0 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=2, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] - │ ProjectionExec: expr=[o_year@0 as o_year, sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END)@1 / sum(all_nations.volume)@2 as mkt_share], metrics=[output_rows=2, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)], metrics=[output_rows=2, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ [Stage 7] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] + │ FilterExec: r_name@1 = AMERICA, projection=[r_regionkey@0], metrics=[output_rows=16, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/region/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/13.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = AMERICA, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= AMERICA AND AMERICA <= r_name_max@1, required_guarantees=[r_name in (AMERICA)], metrics=[output_rows=80, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=12, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([o_year@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)], metrics=[output_rows=12, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ ProjectionExec: expr=[date_part(YEAR, o_orderdate@2) as o_year, l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as volume, n_name@3 as nation], metrics=[output_rows=1155072, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1155072, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@3)], projection=[l_extendedprice@1, l_discount@2, o_orderdate@3, n_name@5], metrics=[output_rows=1155072, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=] - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, o_orderdate@3 as o_orderdate, n_regionkey@4 as n_regionkey, n_name@0 as n_name], metrics=[output_rows=365824, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=365824, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_name@1, l_extendedprice@2, l_discount@3, o_orderdate@5, n_regionkey@6], metrics=[output_rows=365824, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, s_nationkey@3 as s_nationkey, o_orderdate@4 as o_orderdate, n_regionkey@0 as n_regionkey], metrics=[output_rows=22864, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=22864, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@4)], projection=[n_regionkey@1, l_extendedprice@2, l_discount@3, s_nationkey@4, o_orderdate@5], metrics=[output_rows=22864, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1429, elapsed_compute=] - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_custkey@3, c_custkey@0)], projection=[l_extendedprice@0, l_discount@1, s_nationkey@2, o_orderdate@4, c_nationkey@6], metrics=[output_rows=1429, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=3, metrics=[] - │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1429, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([o_custkey@3], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@4 as s_nationkey, o_custkey@0 as o_custkey, o_orderdate@1 as o_orderdate], metrics=[output_rows=1429, elapsed_compute=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1429, elapsed_compute=] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_custkey@1, o_orderdate@2, l_extendedprice@4, l_discount@5, s_nationkey@6], metrics=[output_rows=1429, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] + │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=45624, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([o_orderkey@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=45624, elapsed_compute=] + │ FilterExec: o_orderdate@2 >= 1995-01-01 AND o_orderdate@2 <= 1996-12-31, metrics=[output_rows=45624, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1995-01-01 AND o_orderdate@2 <= 1996-12-31, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1995-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 <= 1996-12-31, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] - │ FilterExec: r_name@1 = AMERICA, projection=[r_regionkey@0], metrics=[output_rows=16, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/region/1.parquet, /testdata/tpch/explain_analyze_sf0.1/region/10.parquet, /testdata/tpch/explain_analyze_sf0.1/region/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/12.parquet, /testdata/tpch/explain_analyze_sf0.1/region/13.parquet, /testdata/tpch/explain_analyze_sf0.1/region/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/15.parquet, /testdata/tpch/explain_analyze_sf0.1/region/16.parquet, /testdata/tpch/explain_analyze_sf0.1/region/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/3.parquet, /testdata/tpch/explain_analyze_sf0.1/region/4.parquet, /testdata/tpch/explain_analyze_sf0.1/region/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/region/6.parquet, /testdata/tpch/explain_analyze_sf0.1/region/7.parquet, /testdata/tpch/explain_analyze_sf0.1/region/8.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = AMERICA, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= AMERICA AND AMERICA <= r_name_max@1, required_guarantees=[r_name in (AMERICA)], metrics=[output_rows=80, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1429, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([o_custkey@3], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@4 as s_nationkey, o_custkey@0 as o_custkey, o_orderdate@1 as o_orderdate], metrics=[output_rows=1429, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1429, elapsed_compute=] - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_custkey@1, o_orderdate@2, l_extendedprice@4, l_discount@5, s_nationkey@6], metrics=[output_rows=1429, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] - │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=45624, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([o_orderkey@0], 18), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=45624, elapsed_compute=] - │ FilterExec: o_orderdate@2 >= 1995-01-01 AND o_orderdate@2 <= 1996-12-31, metrics=[output_rows=45624, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1995-01-01 AND o_orderdate@2 <= 1996-12-31, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1995-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 <= 1996-12-31, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=4485, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([l_orderkey@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@0 as s_nationkey], metrics=[output_rows=4485, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=4485, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_nationkey@1, l_orderkey@2, l_extendedprice@4, l_discount@5], metrics=[output_rows=4485, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=4485, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@1, l_suppkey@3, l_extendedprice@4, l_discount@5], metrics=[output_rows=4485, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=147, elapsed_compute=] - │ [Stage 3] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_partkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=147, elapsed_compute=] - │ FilterExec: p_type@1 = ECONOMY ANODIZED STEEL, projection=[p_partkey@0], metrics=[output_rows=147, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_type], file_type=parquet, predicate=p_type@1 = ECONOMY ANODIZED STEEL, pruning_predicate=p_type_null_count@2 != row_count@3 AND p_type_min@0 <= ECONOMY ANODIZED STEEL AND ECONOMY ANODIZED STEEL <= p_type_max@1, required_guarantees=[p_type in (ECONOMY ANODIZED STEEL)], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=15000, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([c_custkey@0], 6), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + ┌───── Stage 4 ── Tasks: t0:[p0..p17] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=4485, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([l_orderkey@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@0 as s_nationkey], metrics=[output_rows=4485, elapsed_compute=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=4485, elapsed_compute=] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_nationkey@1, l_orderkey@2, l_extendedprice@4, l_discount@5], metrics=[output_rows=4485, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ s_nationkey@1 >= 0 AND s_nationkey@1 <= 24 ], pruning_predicate=s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 0 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 24, required_guarantees=[], metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=4485, elapsed_compute=] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@1, l_suppkey@3, l_extendedprice@4, l_discount@5], metrics=[output_rows=4485, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=147, elapsed_compute=] + │ [Stage 3] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_partkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=147, elapsed_compute=] + │ FilterExec: p_type@1 = ECONOMY ANODIZED STEEL, projection=[p_partkey@0], metrics=[output_rows=147, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/13.parquet], ...]}, projection=[p_partkey, p_type], file_type=parquet, predicate=p_type@1 = ECONOMY ANODIZED STEEL, pruning_predicate=p_type_null_count@2 != row_count@3 AND p_type_min@0 <= ECONOMY ANODIZED STEEL AND ECONOMY ANODIZED STEEL <= p_type_max@1, required_guarantees=[p_type in (ECONOMY ANODIZED STEEL)], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=15000, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([c_custkey@0], 6), input_partitions=4, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/13.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ c_custkey@0 >= 154 AND c_custkey@0 <= 14980 OR c_custkey@0 >= 121 AND c_custkey@0 <= 14917 OR c_custkey@0 >= 34 AND c_custkey@0 <= 14996 OR c_custkey@0 >= 118 AND c_custkey@0 <= 14977 OR c_custkey@0 >= 89 AND c_custkey@0 <= 14713 OR c_custkey@0 >= 4 AND c_custkey@0 <= 14968 ] AND DynamicFilter [ c_nationkey@1 >= 0 AND c_nationkey@1 <= 24 ], pruning_predicate=(c_custkey_null_count@1 != row_count@2 AND c_custkey_max@0 >= 154 AND c_custkey_null_count@1 != row_count@2 AND c_custkey_min@3 <= 14980 OR c_custkey_null_count@1 != row_count@2 AND c_custkey_max@0 >= 121 AND c_custkey_null_count@1 != row_count@2 AND c_custkey_min@3 <= 14917 OR c_custkey_null_count@1 != row_count@2 AND c_custkey_max@0 >= 34 AND c_custkey_null_count@1 != row_count@2 AND c_custkey_min@3 <= 14996 OR c_custkey_null_count@1 != row_count@2 AND c_custkey_max@0 >= 118 AND c_custkey_null_count@1 != row_count@2 AND c_custkey_min@3 <= 14977 OR c_custkey_null_count@1 != row_count@2 AND c_custkey_max@0 >= 89 AND c_custkey_null_count@1 != row_count@2 AND c_custkey_min@3 <= 14713 OR c_custkey_null_count@1 != row_count@2 AND c_custkey_max@0 >= 4 AND c_custkey_null_count@1 != row_count@2 AND c_custkey_min@3 <= 14968) AND c_nationkey_null_count@5 != row_count@2 AND c_nationkey_max@4 >= 0 AND c_nationkey_null_count@5 != row_count@2 AND c_nationkey_min@6 <= 24, required_guarantees=[], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + └────────────────────────────────────────────────── "#); Ok(()) } @@ -470,69 +428,63 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [nation@0 ASC NULLS LAST, o_year@1 DESC], metrics=[output_rows=175, elapsed_compute=, output_bytes=B] - │ [Stage 7] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] + │ SortExec: expr=[nation@0 ASC NULLS LAST, o_year@1 DESC], preserve_partitioning=[true], metrics=[output_rows=175, elapsed_compute=, output_bytes=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] + │ ProjectionExec: expr=[nation@0 as nation, o_year@1 as o_year, sum(profit.amount)@2 as sum_profit], metrics=[output_rows=175, elapsed_compute=, output_bytes=B] + │ AggregateExec: mode=FinalPartitioned, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)], metrics=[output_rows=175, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1050, elapsed_compute=, output_bytes=B] + │ RepartitionExec: partitioning=Hash([nation@0, o_year@1], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)], metrics=[output_rows=1050, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=, reduction_factor=0.2% (1050/514560)] + │ ProjectionExec: expr=[n_name@0 as nation, date_part(YEAR, o_orderdate@5) as o_year, l_extendedprice@2 * (Some(1),20,0 - l_discount@3) - ps_supplycost@4 * l_quantity@1 as amount], metrics=[output_rows=514560, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=514560, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[n_name@1, l_quantity@2, l_extendedprice@3, l_discount@4, ps_supplycost@6, o_orderdate@7], metrics=[output_rows=514560, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=, output_bytes=B] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, metrics=[output_rows=400, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ ProjectionExec: expr=[l_quantity@1 as l_quantity, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@4 as s_nationkey, ps_supplycost@5 as ps_supplycost, o_orderdate@0 as o_orderdate], metrics=[output_rows=32160, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32160, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, l_quantity@3, l_extendedprice@4, l_discount@5, s_nationkey@6, ps_supplycost@7], metrics=[output_rows=32160, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] + │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ SortExec: expr=[nation@0 ASC NULLS LAST, o_year@1 DESC], preserve_partitioning=[true], metrics=[output_rows=175, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] - │ ProjectionExec: expr=[nation@0 as nation, o_year@1 as o_year, sum(profit.amount)@2 as sum_profit], metrics=[output_rows=175, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)], metrics=[output_rows=175, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=150000, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([o_orderkey@0], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:..], ...]}, projection=[o_orderkey, o_orderdate], file_type=parquet, metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1050, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([nation@0, o_year@1], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)], metrics=[output_rows=1050, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ ProjectionExec: expr=[n_name@0 as nation, date_part(YEAR, o_orderdate@5) as o_year, l_extendedprice@2 * (Some(1),20,0 - l_discount@3) - ps_supplycost@4 * l_quantity@1 as amount], metrics=[output_rows=514560, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=514560, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[n_name@1, l_quantity@2, l_extendedprice@3, l_discount@4, ps_supplycost@6, o_orderdate@7], metrics=[output_rows=514560, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ ProjectionExec: expr=[l_quantity@1 as l_quantity, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@4 as s_nationkey, ps_supplycost@5 as ps_supplycost, o_orderdate@0 as o_orderdate], metrics=[output_rows=32160, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32160, elapsed_compute=] - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, l_quantity@3, l_extendedprice@4, l_discount@5, s_nationkey@6, ps_supplycost@7], metrics=[output_rows=32160, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] - │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32160, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([l_orderkey@0], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32160, elapsed_compute=] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_suppkey@2, ps_suppkey@1), (l_partkey@1, ps_partkey@0)], projection=[l_orderkey@0, l_quantity@3, l_extendedprice@4, l_discount@5, s_nationkey@6, ps_supplycost@9], metrics=[output_rows=32160, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] + │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p23] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32160, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([l_suppkey@2, l_partkey@1], 24), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_partkey@2 as l_partkey, l_suppkey@3 as l_suppkey, l_quantity@4 as l_quantity, l_extendedprice@5 as l_extendedprice, l_discount@6 as l_discount, s_nationkey@0 as s_nationkey], metrics=[output_rows=32160, elapsed_compute=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32160, elapsed_compute=] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@2)], projection=[s_nationkey@1, l_orderkey@2, l_partkey@3, l_suppkey@4, l_quantity@5, l_extendedprice@6, l_discount@7], metrics=[output_rows=32160, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ s_nationkey@1 >= 0 AND s_nationkey@1 <= 24 ], pruning_predicate=s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 0 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 24, required_guarantees=[], metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32160, elapsed_compute=] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@1, l_partkey@2, l_suppkey@3, l_quantity@4, l_extendedprice@5, l_discount@6], metrics=[output_rows=32160, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=1075, elapsed_compute=] + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ l_orderkey@0 >= 32 AND l_orderkey@0 <= 599974 OR l_orderkey@0 >= 65 AND l_orderkey@0 <= 599972 OR l_orderkey@0 >= 5 AND l_orderkey@0 <= 599973 OR l_orderkey@0 >= 66 AND l_orderkey@0 <= 599971 OR l_orderkey@0 >= 3 AND l_orderkey@0 <= 599975 OR l_orderkey@0 >= 1 AND l_orderkey@0 <= 600000 ], pruning_predicate=l_orderkey_null_count@1 != row_count@2 AND l_orderkey_max@0 >= 32 AND l_orderkey_null_count@1 != row_count@2 AND l_orderkey_min@3 <= 599974 OR l_orderkey_null_count@1 != row_count@2 AND l_orderkey_max@0 >= 65 AND l_orderkey_null_count@1 != row_count@2 AND l_orderkey_min@3 <= 599972 OR l_orderkey_null_count@1 != row_count@2 AND l_orderkey_max@0 >= 5 AND l_orderkey_null_count@1 != row_count@2 AND l_orderkey_min@3 <= 599973 OR l_orderkey_null_count@1 != row_count@2 AND l_orderkey_max@0 >= 66 AND l_orderkey_null_count@1 != row_count@2 AND l_orderkey_min@3 <= 599971 OR l_orderkey_null_count@1 != row_count@2 AND l_orderkey_max@0 >= 3 AND l_orderkey_null_count@1 != row_count@2 AND l_orderkey_min@3 <= 599975 OR l_orderkey_null_count@1 != row_count@2 AND l_orderkey_max@0 >= 1 AND l_orderkey_null_count@1 != row_count@2 AND l_orderkey_min@3 <= 600000, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=150000, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([o_orderkey@0], 6), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_orderdate], file_type=parquet, metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32160, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([l_orderkey@0], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32160, elapsed_compute=] - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_suppkey@2, ps_suppkey@1), (l_partkey@1, ps_partkey@0)], projection=[l_orderkey@0, l_quantity@3, l_extendedprice@4, l_discount@5, s_nationkey@6, ps_supplycost@9], metrics=[output_rows=32160, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] - │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1075, elapsed_compute=] + │ FilterExec: p_name@1 LIKE %green%, projection=[p_partkey@0], metrics=[output_rows=1075, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/13.parquet], ...]}, projection=[p_partkey, p_name], file_type=parquet, predicate=p_name@1 LIKE %green%, metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p23] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32160, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([l_suppkey@2, l_partkey@1], 24), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_partkey@2 as l_partkey, l_suppkey@3 as l_suppkey, l_quantity@4 as l_quantity, l_extendedprice@5 as l_extendedprice, l_discount@6 as l_discount, s_nationkey@0 as s_nationkey], metrics=[output_rows=32160, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32160, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@2)], projection=[s_nationkey@1, l_orderkey@2, l_partkey@3, l_suppkey@4, l_quantity@5, l_extendedprice@6, l_discount@7], metrics=[output_rows=32160, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=32160, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@1, l_partkey@2, l_suppkey@3, l_quantity@4, l_extendedprice@5, l_discount@6], metrics=[output_rows=32160, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=1075, elapsed_compute=] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1075, elapsed_compute=] - │ FilterExec: p_name@1 LIKE %green%, projection=[p_partkey@0], metrics=[output_rows=1075, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_name], file_type=parquet, predicate=p_name@1 LIKE %green%, metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=80000, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([ps_suppkey@1, ps_partkey@0], 24), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=80000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=80000, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([ps_suppkey@1, ps_partkey@0], 24), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=80000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + └────────────────────────────────────────────────── "); Ok(()) } @@ -543,42 +495,36 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [revenue@2 DESC], metrics=[output_rows=3767, elapsed_compute=, output_bytes=B] - │ [Stage 3] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] + │ SortExec: expr=[revenue@2 DESC], preserve_partitioning=[true], metrics=[output_rows=3767, elapsed_compute=, output_bytes=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] + │ ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@7 as revenue, c_acctbal@2 as c_acctbal, n_name@4 as n_name, c_address@5 as c_address, c_phone@3 as c_phone, c_comment@6 as c_comment], metrics=[output_rows=3767, elapsed_compute=, output_bytes=B] + │ AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@2 as c_acctbal, c_phone@3 as c_phone, n_name@4 as n_name, c_address@5 as c_address, c_comment@6 as c_comment], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=3767, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=4648, elapsed_compute=, output_bytes=B] + │ RepartitionExec: partitioning=Hash([c_custkey@0, c_name@1, c_acctbal@2, c_phone@3, n_name@4, c_address@5, c_comment@6], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@4 as c_acctbal, c_phone@3 as c_phone, n_name@8 as n_name, c_address@2 as c_address, c_comment@5 as c_comment], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=4648, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=, reduction_factor=2.5% (4648/183024)] + │ ProjectionExec: expr=[c_custkey@1 as c_custkey, c_name@2 as c_name, c_address@3 as c_address, c_phone@4 as c_phone, c_acctbal@5 as c_acctbal, c_comment@6 as c_comment, l_extendedprice@7 as l_extendedprice, l_discount@8 as l_discount, n_name@0 as n_name], metrics=[output_rows=183024, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=183024, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@3)], projection=[n_name@1, c_custkey@2, c_name@3, c_address@4, c_phone@6, c_acctbal@7, c_comment@8, l_extendedprice@9, l_discount@10], metrics=[output_rows=183024, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=, output_bytes=B] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, metrics=[output_rows=400, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=11439, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@7, l_orderkey@0)], projection=[c_custkey@0, c_name@1, c_address@2, c_nationkey@3, c_phone@4, c_acctbal@5, c_comment@6, l_extendedprice@9, l_discount@10], metrics=[output_rows=11439, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=5677, elapsed_compute=, output_bytes=B] + │ ProjectionExec: expr=[c_custkey@1 as c_custkey, c_name@2 as c_name, c_address@3 as c_address, c_nationkey@4 as c_nationkey, c_phone@5 as c_phone, c_acctbal@6 as c_acctbal, c_comment@7 as c_comment, o_orderkey@0 as o_orderkey], metrics=[output_rows=5677, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5677, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_custkey@1, c_custkey@0)], projection=[o_orderkey@0, c_custkey@2, c_name@3, c_address@4, c_nationkey@5, c_phone@6, c_acctbal@7, c_comment@8], metrics=[output_rows=5677, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=5677, elapsed_compute=, output_bytes=B] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=24, input_tasks=4, metrics=[] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_comment], file_type=parquet, predicate=DynamicFilter [ c_custkey@0 >= 2 AND c_custkey@0 <= 14992 ] AND DynamicFilter [ c_nationkey@3 >= 0 AND c_nationkey@3 <= 24 ], pruning_predicate=c_custkey_null_count@1 != row_count@2 AND c_custkey_max@0 >= 2 AND c_custkey_null_count@1 != row_count@2 AND c_custkey_min@3 <= 14992 AND c_nationkey_null_count@5 != row_count@2 AND c_nationkey_max@4 >= 0 AND c_nationkey_null_count@5 != row_count@2 AND c_nationkey_min@6 <= 24, required_guarantees=[], metrics=[output_rows=15000, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=148301, elapsed_compute=, output_bytes=B] + │ FilterExec: l_returnflag@3 = R, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2], metrics=[output_rows=148301, elapsed_compute=, output_bytes=B, selectivity=25% (148301/600572)] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_extendedprice, l_discount, l_returnflag], file_type=parquet, predicate=l_returnflag@3 = R AND DynamicFilter [ l_orderkey@0 >= 3 AND l_orderkey@0 <= 599879 ], pruning_predicate=l_returnflag_null_count@2 != row_count@3 AND l_returnflag_min@0 <= R AND R <= l_returnflag_max@1 AND l_orderkey_null_count@5 != row_count@3 AND l_orderkey_max@4 >= 3 AND l_orderkey_null_count@5 != row_count@3 AND l_orderkey_min@6 <= 599879, required_guarantees=[l_returnflag in (R)], metrics=[output_rows=600572, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ SortExec: expr=[revenue@2 DESC], preserve_partitioning=[true], metrics=[output_rows=3767, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] - │ ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@7 as revenue, c_acctbal@2 as c_acctbal, n_name@4 as n_name, c_address@5 as c_address, c_phone@3 as c_phone, c_comment@6 as c_comment], metrics=[output_rows=3767, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@2 as c_acctbal, c_phone@3 as c_phone, n_name@4 as n_name, c_address@5 as c_address, c_comment@6 as c_comment], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=3767, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] t3:[p18..p23] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5677, elapsed_compute=] + │ FilterExec: o_orderdate@2 >= 1993-10-01 AND o_orderdate@2 < 1994-01-01, projection=[o_orderkey@0, o_custkey@1], metrics=[output_rows=5677, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1993-10-01 AND o_orderdate@2 < 1994-01-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-10-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1994-01-01, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=4648, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([c_custkey@0, c_name@1, c_acctbal@2, c_phone@3, n_name@4, c_address@5, c_comment@6], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@4 as c_acctbal, c_phone@3 as c_phone, n_name@8 as n_name, c_address@2 as c_address, c_comment@5 as c_comment], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=4648, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ ProjectionExec: expr=[c_custkey@1 as c_custkey, c_name@2 as c_name, c_address@3 as c_address, c_phone@4 as c_phone, c_acctbal@5 as c_acctbal, c_comment@6 as c_comment, l_extendedprice@7 as l_extendedprice, l_discount@8 as l_discount, n_name@0 as n_name], metrics=[output_rows=183024, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=183024, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@3)], projection=[n_name@1, c_custkey@2, c_name@3, c_address@4, c_phone@6, c_acctbal@7, c_comment@8, l_extendedprice@9, l_discount@10], metrics=[output_rows=183024, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=400, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=11439, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@7, l_orderkey@0)], projection=[c_custkey@0, c_name@1, c_address@2, c_nationkey@3, c_phone@4, c_acctbal@5, c_comment@6, l_extendedprice@9, l_discount@10], metrics=[output_rows=11439, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=5677, elapsed_compute=] - │ ProjectionExec: expr=[c_custkey@1 as c_custkey, c_name@2 as c_name, c_address@3 as c_address, c_nationkey@4 as c_nationkey, c_phone@5 as c_phone, c_acctbal@6 as c_acctbal, c_comment@7 as c_comment, o_orderkey@0 as o_orderkey], metrics=[output_rows=5677, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5677, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_custkey@1, c_custkey@0)], projection=[o_orderkey@0, c_custkey@2, c_name@3, c_address@4, c_nationkey@5, c_phone@6, c_acctbal@7, c_comment@8], metrics=[output_rows=5677, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=5677, elapsed_compute=] - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_comment], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=148301, elapsed_compute=] - │ FilterExec: l_returnflag@3 = R, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2], metrics=[output_rows=148301, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_extendedprice, l_discount, l_returnflag], file_type=parquet, predicate=l_returnflag@3 = R AND DynamicFilter [ empty ], pruning_predicate=l_returnflag_null_count@2 != row_count@3 AND l_returnflag_min@0 <= R AND R <= l_returnflag_max@1, required_guarantees=[l_returnflag in (R)], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5677, elapsed_compute=] - │ FilterExec: o_orderdate@2 >= 1993-10-01 AND o_orderdate@2 < 1994-01-01, projection=[o_orderkey@0, o_custkey@1], metrics=[output_rows=5677, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1993-10-01 AND o_orderdate@2 < 1994-01-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-10-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1994-01-01, required_guarantees=[], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── "); Ok(()) } @@ -598,8 +544,8 @@ mod tests { │ AggregateExec: mode=Partial, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)], metrics=[output_rows=6, elapsed_compute=, output_bytes=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=64000, elapsed_compute=, output_bytes=B] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[ps_availqty@1, ps_supplycost@2], metrics=[output_rows=64000, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=, output_bytes=] - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] + │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=, output_bytes=B] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] │ ProjectionExec: expr=[ps_availqty@1 as ps_availqty, ps_supplycost@2 as ps_supplycost, s_nationkey@0 as s_nationkey], metrics=[output_rows=80000, elapsed_compute=, output_bytes=B] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=80000, elapsed_compute=, output_bytes=B] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@0)], projection=[s_nationkey@1, ps_availqty@3, ps_supplycost@4], metrics=[output_rows=80000, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] @@ -613,8 +559,8 @@ mod tests { │ AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)], metrics=[output_rows=3716, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=, reduction_factor=5.8% (3716/64000)] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=64000, elapsed_compute=, output_bytes=B] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[ps_partkey@1, ps_availqty@2, ps_supplycost@3], metrics=[output_rows=64000, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=, output_bytes=] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] + │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=, output_bytes=B] + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_availqty@2 as ps_availqty, ps_supplycost@3 as ps_supplycost, s_nationkey@0 as s_nationkey], metrics=[output_rows=80000, elapsed_compute=, output_bytes=B] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=80000, elapsed_compute=, output_bytes=B] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@1)], projection=[s_nationkey@1, ps_partkey@2, ps_availqty@4, ps_supplycost@5], metrics=[output_rows=80000, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] @@ -622,17 +568,17 @@ mod tests { │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ s_nationkey@1 >= 7 AND s_nationkey@1 <= 7 ], pruning_predicate=s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 7 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 7, required_guarantees=[], metrics=[output_rows=1000, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ ps_suppkey@1 >= 1 AND ps_suppkey@1 <= 1000 ], pruning_predicate=ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000, required_guarantees=[], metrics=[output_rows=80000, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] │ FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0], metrics=[output_rows=16, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (GERMANY)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (GERMANY)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] │ FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0], metrics=[output_rows=16, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (GERMANY)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (GERMANY)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -650,30 +596,30 @@ mod tests { │ SortExec: expr=[l_shipmode@0 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=2, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] │ ProjectionExec: expr=[l_shipmode@0 as l_shipmode, sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@1 as high_line_count, sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@2 as low_line_count], metrics=[output_rows=2, elapsed_compute=] │ AggregateExec: mode=FinalPartitioned, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)], metrics=[output_rows=2, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=3, metrics=[] + │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=36, elapsed_compute=] + ┌───── Stage 3 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=48, elapsed_compute=] │ RepartitionExec: partitioning=Hash([l_shipmode@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)], metrics=[output_rows=36, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ AggregateExec: mode=Partial, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)], metrics=[output_rows=48, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=3155, elapsed_compute=] │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_orderkey@0, o_orderkey@0)], projection=[l_shipmode@1, o_orderpriority@3], metrics=[output_rows=3155, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] + ┌───── Stage 1 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=3155, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([l_orderkey@0], 18), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([l_orderkey@0], 24), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=3155, elapsed_compute=] │ FilterExec: (l_shipmode@4 = MAIL OR l_shipmode@4 = SHIP) AND l_receiptdate@3 > l_commitdate@2 AND l_shipdate@1 < l_commitdate@2 AND l_receiptdate@3 >= 1994-01-01 AND l_receiptdate@3 < 1995-01-01, projection=[l_orderkey@0, l_shipmode@4], metrics=[output_rows=3155, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_shipdate, l_commitdate, l_receiptdate, l_shipmode], file_type=parquet, predicate=(l_shipmode@4 = MAIL OR l_shipmode@4 = SHIP) AND l_receiptdate@3 > l_commitdate@2 AND l_shipdate@1 < l_commitdate@2 AND l_receiptdate@3 >= 1994-01-01 AND l_receiptdate@3 < 1995-01-01, pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= MAIL AND MAIL <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1) AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_max@4 >= 1994-01-01 AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_min@6 < 1995-01-01, required_guarantees=[l_shipmode in (MAIL, SHIP)], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:..], ...]}, projection=[l_orderkey, l_shipdate, l_commitdate, l_receiptdate, l_shipmode], file_type=parquet, predicate=(l_shipmode@4 = MAIL OR l_shipmode@4 = SHIP) AND l_receiptdate@3 > l_commitdate@2 AND l_shipdate@1 < l_commitdate@2 AND l_receiptdate@3 >= 1994-01-01 AND l_receiptdate@3 < 1995-01-01, pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= MAIL AND MAIL <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1) AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_max@4 >= 1994-01-01 AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_min@6 < 1995-01-01, required_guarantees=[l_shipmode in (MAIL, SHIP)], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] + ┌───── Stage 2 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=150000, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([o_orderkey@0], 18), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_orderpriority], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ RepartitionExec: partitioning=Hash([o_orderkey@0], 24), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:..], ...]}, projection=[o_orderkey, o_orderpriority], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "#); Ok(()) @@ -685,39 +631,30 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [custdist@1 DESC, c_count@0 DESC], metrics=[output_rows=37, elapsed_compute=, output_bytes=] - │ [Stage 4] => NetworkCoalesceExec: output_partitions=12, input_tasks=2, metrics=[] + │ SortExec: expr=[custdist@1 DESC, c_count@0 DESC], preserve_partitioning=[true], metrics=[output_rows=37, elapsed_compute=, output_bytes=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] + │ ProjectionExec: expr=[c_count@0 as c_count, count(Int64(1))@1 as custdist], metrics=[output_rows=37, elapsed_compute=, output_bytes=B] + │ AggregateExec: mode=FinalPartitioned, gby=[c_count@0 as c_count], aggr=[count(Int64(1))], metrics=[output_rows=37, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=205, elapsed_compute=, output_bytes=B] + │ RepartitionExec: partitioning=Hash([c_count@0], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[c_count@0 as c_count], aggr=[count(Int64(1))], metrics=[output_rows=205, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=, reduction_factor=1.4% (205/15000)] + │ ProjectionExec: expr=[count(orders.o_orderkey)@1 as c_count], metrics=[output_rows=15000, elapsed_compute=, output_bytes=B] + │ AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey], aggr=[count(orders.o_orderkey)], metrics=[output_rows=15000, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=15000, elapsed_compute=, output_bytes=B] + │ RepartitionExec: partitioning=Hash([c_custkey@0], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey], aggr=[count(orders.o_orderkey)], metrics=[output_rows=15000, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=, reduction_factor=9.8% (15000/153318)] + │ ProjectionExec: expr=[c_custkey@1 as c_custkey, o_orderkey@0 as o_orderkey], metrics=[output_rows=153318, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=153318, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(o_custkey@1, c_custkey@0)], projection=[o_orderkey@0, c_custkey@2], metrics=[output_rows=153318, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=148318, elapsed_compute=, output_bytes=B] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=24, input_tasks=4, metrics=[] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey], file_type=parquet, metrics=[output_rows=15000, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] - │ SortExec: expr=[custdist@1 DESC, c_count@0 DESC], preserve_partitioning=[true], metrics=[output_rows=37, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] - │ ProjectionExec: expr=[c_count@0 as c_count, count(Int64(1))@1 as custdist], metrics=[output_rows=37, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[c_count@0 as c_count], aggr=[count(Int64(1))], metrics=[output_rows=37, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=3, metrics=[] + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] t3:[p18..p23] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=148318, elapsed_compute=] + │ FilterExec: o_comment@2 NOT LIKE %special%requests%, projection=[o_orderkey@0, o_custkey@1], metrics=[output_rows=148318, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_comment], file_type=parquet, predicate=o_comment@2 NOT LIKE %special%requests%, metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p11] t1:[p0..p11] t2:[p0..p11] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=568, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([c_count@0], 12), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[c_count@0 as c_count], aggr=[count(Int64(1))], metrics=[output_rows=568, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ ProjectionExec: expr=[count(orders.o_orderkey)@1 as c_count], metrics=[output_rows=15000, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey], aggr=[count(orders.o_orderkey)], metrics=[output_rows=15000, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=15000, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([c_custkey@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey], aggr=[count(orders.o_orderkey)], metrics=[output_rows=15000, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ ProjectionExec: expr=[c_custkey@1 as c_custkey, o_orderkey@0 as o_orderkey], metrics=[output_rows=153318, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=153318, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(o_custkey@1, c_custkey@0)], projection=[o_orderkey@0, c_custkey@2], metrics=[output_rows=153318, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=148318, elapsed_compute=] - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey], file_type=parquet, metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=148318, elapsed_compute=] - │ FilterExec: o_comment@2 NOT LIKE %special%requests%, projection=[o_orderkey@0, o_custkey@1], metrics=[output_rows=148318, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_comment], file_type=parquet, predicate=o_comment@2 NOT LIKE %special%requests%, metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── "); Ok(()) } @@ -742,17 +679,17 @@ mod tests { └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=20000, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([p_partkey@0], 24), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_type], file_type=parquet, metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ RepartitionExec: partitioning=Hash([p_partkey@0], 24), input_partitions=4, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/13.parquet], ...]}, projection=[p_partkey, p_type], file_type=parquet, metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=7630, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([l_partkey@0], 24), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([l_partkey@0], 24), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=7630, elapsed_compute=] │ FilterExec: l_shipdate@3 >= 1995-09-01 AND l_shipdate@3 < 1995-10-01, projection=[l_partkey@0, l_extendedprice@1, l_discount@2], metrics=[output_rows=7630, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_partkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1995-09-01 AND l_shipdate@3 < 1995-10-01 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-09-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-10-01, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:..], ...]}, projection=[l_partkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1995-09-01 AND l_shipdate@3 < 1995-10-01 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-09-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-10-01, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "#); Ok(()) @@ -785,22 +722,22 @@ mod tests { │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p11] t1:[p0..p11] t2:[p0..p11] t3:[p0..p11] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5825, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([l_suppkey@0], 12), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=5825, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=12179, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([l_suppkey@0], 12), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=12179, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=22830, elapsed_compute=] │ FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2], metrics=[output_rows=22830, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:..], ...]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5825, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([l_suppkey@0], 6), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=5825, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=12179, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([l_suppkey@0], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)], metrics=[output_rows=12179, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=22830, elapsed_compute=] │ FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2], metrics=[output_rows=22830, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:..], ...]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -812,48 +749,39 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], metrics=[output_rows=2762, elapsed_compute=, output_bytes=B] - │ [Stage 5] => NetworkCoalesceExec: output_partitions=12, input_tasks=2, metrics=[] + │ SortExec: expr=[supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=2762, elapsed_compute=, output_bytes=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] + │ ProjectionExec: expr=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, count(alias1)@3 as supplier_cnt], metrics=[output_rows=2762, elapsed_compute=, output_bytes=B] + │ AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)], metrics=[output_rows=2762, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=8718, elapsed_compute=, output_bytes=B] + │ RepartitionExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)], metrics=[output_rows=8718, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=, reduction_factor=75% (8718/11632)] + │ AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, alias1@3 as alias1], aggr=[], metrics=[output_rows=11632, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=11634, elapsed_compute=, output_bytes=B] + │ RepartitionExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2, alias1@3], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[p_brand@1 as p_brand, p_type@2 as p_type, p_size@3 as p_size, ps_suppkey@0 as alias1], aggr=[], metrics=[output_rows=11634, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=, reduction_factor=100% (11634/11635)] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=11635, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(s_suppkey@0, ps_suppkey@0)], metrics=[output_rows=11635, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=1, elapsed_compute=, output_bytes=] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] + │ ProjectionExec: expr=[ps_suppkey@3 as ps_suppkey, p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], metrics=[output_rows=11644, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=11644, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, ps_partkey@0)], projection=[p_brand@1, p_type@2, p_size@3, ps_suppkey@5], metrics=[output_rows=11644, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=2911, elapsed_compute=, output_bytes=B] + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey], file_type=parquet, predicate=DynamicFilter [ ps_partkey@0 >= 4 AND ps_partkey@0 <= 19994 ], pruning_predicate=ps_partkey_null_count@1 != row_count@2 AND ps_partkey_max@0 >= 4 AND ps_partkey_null_count@1 != row_count@2 AND ps_partkey_min@3 <= 19994, required_guarantees=[], metrics=[output_rows=80000, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] - │ SortExec: expr=[supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=2762, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] - │ ProjectionExec: expr=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, count(alias1)@3 as supplier_cnt], metrics=[output_rows=2762, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)], metrics=[output_rows=2762, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=3, metrics=[] + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1, elapsed_compute=] + │ FilterExec: s_comment@1 LIKE %Customer%Complaints%, projection=[s_suppkey@0], metrics=[output_rows=1, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet], ...]}, projection=[s_suppkey, s_comment], file_type=parquet, predicate=s_comment@1 LIKE %Customer%Complaints%, metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=2911, elapsed_compute=] + │ FilterExec: p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND p_size@3 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]), metrics=[output_rows=2911, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/13.parquet], ...]}, projection=[p_partkey, p_brand, p_type, p_size], file_type=parquet, predicate=p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND p_size@3 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]), pruning_predicate=p_brand_null_count@2 != row_count@3 AND (p_brand_min@0 != Brand#45 OR Brand#45 != p_brand_max@1) AND p_type_null_count@6 != row_count@3 AND (p_type_min@4 NOT LIKE MEDIUM POLISHED% OR p_type_max@5 NOT LIKE MEDIUM POLISHED%) AND (p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 49 AND 49 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 14 AND 14 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 23 AND 23 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 45 AND 45 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 19 AND 19 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 3 AND 3 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 36 AND 36 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 9 AND 9 <= p_size_max@8), required_guarantees=[p_brand not in (Brand#45), p_size in (14, 19, 23, 3, 36, 45, 49, 9)], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p11] t1:[p0..p11] t2:[p0..p11] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=10538, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2], 12), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)], metrics=[output_rows=10538, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, alias1@3 as alias1], aggr=[], metrics=[output_rows=11632, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=11634, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2, alias1@3], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[p_brand@1 as p_brand, p_type@2 as p_type, p_size@3 as p_size, ps_suppkey@0 as alias1], aggr=[], metrics=[output_rows=11634, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=11635, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(s_suppkey@0, ps_suppkey@0)], metrics=[output_rows=11635, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=1, elapsed_compute=] - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ ProjectionExec: expr=[ps_suppkey@3 as ps_suppkey, p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], metrics=[output_rows=11644, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=11644, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, ps_partkey@0)], projection=[p_brand@1, p_type@2, p_size@3, ps_suppkey@5], metrics=[output_rows=11644, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=2911, elapsed_compute=] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=80000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1, elapsed_compute=] - │ FilterExec: s_comment@1 LIKE %Customer%Complaints%, projection=[s_suppkey@0], metrics=[output_rows=1, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_comment], file_type=parquet, predicate=s_comment@1 LIKE %Customer%Complaints%, metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=2911, elapsed_compute=] - │ FilterExec: p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND p_size@3 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]), metrics=[output_rows=2911, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_brand, p_type, p_size], file_type=parquet, predicate=p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND p_size@3 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]), pruning_predicate=p_brand_null_count@2 != row_count@3 AND (p_brand_min@0 != Brand#45 OR Brand#45 != p_brand_max@1) AND p_type_null_count@6 != row_count@3 AND (p_type_min@4 NOT LIKE MEDIUM POLISHED% OR p_type_max@5 NOT LIKE MEDIUM POLISHED%) AND (p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 49 AND 49 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 14 AND 14 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 23 AND 23 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 45 AND 45 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 19 AND 19 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 3 AND 3 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 36 AND 36 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 9 AND 9 <= p_size_max@8), required_guarantees=[p_brand not in (Brand#45), p_size in (14, 19, 23, 3, 36, 45, 49, 9)], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── "); Ok(()) } @@ -865,11 +793,11 @@ mod tests { ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[CAST(sum(lineitem.l_extendedprice)@0 AS Float64) / 7 as avg_yearly], metrics=[output_rows=1, elapsed_compute=, output_bytes=] │ AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice)], metrics=[output_rows=1, elapsed_compute=, output_bytes=] - │ CoalescePartitionsExec, metrics=[output_rows=24, elapsed_compute=, output_bytes=B] - │ [Stage 4] => NetworkCoalesceExec: output_partitions=24, input_tasks=4, metrics=[] + │ CoalescePartitionsExec, metrics=[output_rows=18, elapsed_compute=, output_bytes=B] + │ [Stage 4] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice)], metrics=[output_rows=24, elapsed_compute=] + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice)], metrics=[output_rows=18, elapsed_compute=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=43, elapsed_compute=] │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@2, l_partkey@1)], filter=CAST(l_quantity@0 AS Decimal128(30, 15)) < Float64(0.2) * avg(lineitem.l_quantity)@1, projection=[l_extendedprice@1], metrics=[output_rows=43, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] @@ -877,28 +805,28 @@ mod tests { │ AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)], metrics=[output_rows=20000, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p23] + ┌───── Stage 2 ── Tasks: t0:[p0..p17] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=555, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([p_partkey@2], 24), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ RepartitionExec: partitioning=Hash([p_partkey@2], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] │ ProjectionExec: expr=[l_quantity@1 as l_quantity, l_extendedprice@2 as l_extendedprice, p_partkey@0 as p_partkey], metrics=[output_rows=555, elapsed_compute=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=555, elapsed_compute=] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@0)], projection=[p_partkey@0, l_quantity@2, l_extendedprice@3], metrics=[output_rows=555, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=18, elapsed_compute=] - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_partkey, l_quantity, l_extendedprice], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=18, elapsed_compute=] │ FilterExec: p_brand@1 = Brand#23 AND p_container@2 = MED BOX, projection=[p_partkey@0], metrics=[output_rows=18, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_brand, p_container], file_type=parquet, predicate=p_brand@1 = Brand#23 AND p_container@2 = MED BOX, pruning_predicate=p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5, required_guarantees=[p_brand in (Brand#23), p_container in (MED BOX)], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/13.parquet], ...]}, projection=[p_partkey, p_brand, p_container], file_type=parquet, predicate=p_brand@1 = Brand#23 AND p_container@2 = MED BOX, pruning_predicate=p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5, required_guarantees=[p_brand in (Brand#23), p_container in (MED BOX)], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=118823, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([l_partkey@0], 24), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)], metrics=[output_rows=118823, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_partkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + ┌───── Stage 3 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=270849, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([l_partkey@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)], metrics=[output_rows=270849, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:..], ...]}, projection=[l_partkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -909,66 +837,60 @@ mod tests { let plan = test_tpch_query(18).await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST], metrics=[output_rows=5, elapsed_compute=, output_bytes=B] - │ [Stage 8] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] + │ SortPreservingMergeExec: [o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST], metrics=[output_rows=5, elapsed_compute=, output_bytes=] + │ SortExec: expr=[o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=5, elapsed_compute=, output_bytes=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] + │ AggregateExec: mode=FinalPartitioned, gby=[c_name@0 as c_name, c_custkey@1 as c_custkey, o_orderkey@2 as o_orderkey, o_orderdate@3 as o_orderdate, o_totalprice@4 as o_totalprice], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=5, elapsed_compute=, output_bytes=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5, elapsed_compute=, output_bytes=B] + │ RepartitionExec: partitioning=Hash([c_name@0, c_custkey@1, o_orderkey@2, o_orderdate@3, o_totalprice@4], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[c_name@1 as c_name, c_custkey@0 as c_custkey, o_orderkey@2 as o_orderkey, o_orderdate@4 as o_orderdate, o_totalprice@3 as o_totalprice], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=5, elapsed_compute=, output_bytes=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=, reduction_factor=14% (5/35)] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=35, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(l_orderkey@0, o_orderkey@2)], metrics=[output_rows=35, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=5, elapsed_compute=, output_bytes=] + │ [Stage 2] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=600572, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_orderkey@2, l_orderkey@0)], projection=[c_custkey@0, c_name@1, o_orderkey@2, o_totalprice@3, o_orderdate@4, l_quantity@6], metrics=[output_rows=600572, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] + │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ SortExec: expr=[o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=5, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] - │ AggregateExec: mode=FinalPartitioned, gby=[c_name@0 as c_name, c_custkey@1 as c_custkey, o_orderkey@2 as o_orderkey, o_orderdate@3 as o_orderdate, o_totalprice@4 as o_totalprice], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=5, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ [Stage 7] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5, elapsed_compute=] + │ FilterExec: sum(lineitem.l_quantity)@1 > Some(30000),25,2, projection=[l_orderkey@0], metrics=[output_rows=5, elapsed_compute=] + │ AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=150000, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([c_name@0, c_custkey@1, o_orderkey@2, o_orderdate@3, o_totalprice@4], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[c_name@1 as c_name, c_custkey@0 as c_custkey, o_orderkey@2 as o_orderkey, o_orderdate@4 as o_orderdate, o_totalprice@3 as o_totalprice], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=5, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=35, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(l_orderkey@0, o_orderkey@2)], metrics=[output_rows=35, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=5, elapsed_compute=] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=600572, elapsed_compute=] - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_orderkey@2, l_orderkey@0)], projection=[c_custkey@0, c_name@1, o_orderkey@2, o_totalprice@3, o_orderdate@4, l_quantity@6], metrics=[output_rows=600572, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] - │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] + ┌───── Stage 1 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=150000, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([l_orderkey@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=150000, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:..], ...]}, projection=[l_orderkey, l_quantity], file_type=parquet, metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=5, elapsed_compute=] - │ FilterExec: sum(lineitem.l_quantity)@1 > Some(30000),25,2, projection=[l_orderkey@0], metrics=[output_rows=5, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=150000, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=150000, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([l_orderkey@0], 18), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=150000, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_quantity], file_type=parquet, metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=150000, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([o_orderkey@2], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=150000, elapsed_compute=] - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c_custkey@0, o_custkey@1)], projection=[c_custkey@0, c_name@1, o_orderkey@2, o_totalprice@4, o_orderdate@5], metrics=[output_rows=150000, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] - │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=15000, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([c_custkey@0], 24), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_name], file_type=parquet, metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=150000, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([o_custkey@1], 24), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_totalprice, o_orderdate], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=600572, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([l_orderkey@0], 6), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=150000, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([o_orderkey@2], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=150000, elapsed_compute=] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c_custkey@0, o_custkey@1)], projection=[c_custkey@0, c_name@1, o_orderkey@2, o_totalprice@4, o_orderdate@5], metrics=[output_rows=150000, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] + │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=15000, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([c_custkey@0], 24), input_partitions=4, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/13.parquet], ...]}, projection=[c_custkey, c_name], file_type=parquet, metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=150000, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([o_custkey@1], 24), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_totalprice, o_orderdate], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=600572, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([l_orderkey@0], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:..], ...]}, projection=[l_orderkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ l_orderkey@0 >= 32 AND l_orderkey@0 <= 599974 OR l_orderkey@0 >= 65 AND l_orderkey@0 <= 599972 OR l_orderkey@0 >= 5 AND l_orderkey@0 <= 599973 OR l_orderkey@0 >= 66 AND l_orderkey@0 <= 599971 OR l_orderkey@0 >= 3 AND l_orderkey@0 <= 599975 OR l_orderkey@0 >= 1 AND l_orderkey@0 <= 600000 ], pruning_predicate=l_orderkey_null_count@1 != row_count@2 AND l_orderkey_max@0 >= 32 AND l_orderkey_null_count@1 != row_count@2 AND l_orderkey_min@3 <= 599974 OR l_orderkey_null_count@1 != row_count@2 AND l_orderkey_max@0 >= 65 AND l_orderkey_null_count@1 != row_count@2 AND l_orderkey_min@3 <= 599972 OR l_orderkey_null_count@1 != row_count@2 AND l_orderkey_max@0 >= 5 AND l_orderkey_null_count@1 != row_count@2 AND l_orderkey_min@3 <= 599973 OR l_orderkey_null_count@1 != row_count@2 AND l_orderkey_max@0 >= 66 AND l_orderkey_null_count@1 != row_count@2 AND l_orderkey_min@3 <= 599971 OR l_orderkey_null_count@1 != row_count@2 AND l_orderkey_max@0 >= 3 AND l_orderkey_null_count@1 != row_count@2 AND l_orderkey_min@3 <= 599975 OR l_orderkey_null_count@1 != row_count@2 AND l_orderkey_max@0 >= 1 AND l_orderkey_null_count@1 != row_count@2 AND l_orderkey_min@3 <= 600000, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + └────────────────────────────────────────────────── "); Ok(()) } @@ -985,16 +907,16 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=10, elapsed_compute=, output_bytes=B] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN ([SM CASE, SM BOX, SM PACK, SM PKG]) AND l_quantity@0 >= Some(100),15,2 AND l_quantity@0 <= Some(1100),15,2 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([MED BAG, MED BOX, MED PKG, MED PACK]) AND l_quantity@0 >= Some(1000),15,2 AND l_quantity@0 <= Some(2000),15,2 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([LG CASE, LG BOX, LG PACK, LG PKG]) AND l_quantity@0 >= Some(2000),15,2 AND l_quantity@0 <= Some(3000),15,2 AND p_size@2 <= 15, projection=[l_extendedprice@6, l_discount@7], metrics=[output_rows=10, elapsed_compute=, output_bytes=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=42, elapsed_compute=, output_bytes=B] - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=12635, elapsed_compute=, output_bytes=B] │ FilterExec: (l_quantity@1 >= Some(100),15,2 AND l_quantity@1 <= Some(1100),15,2 OR l_quantity@1 >= Some(1000),15,2 AND l_quantity@1 <= Some(2000),15,2 OR l_quantity@1 >= Some(2000),15,2 AND l_quantity@1 <= Some(3000),15,2) AND (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON, projection=[l_partkey@0, l_quantity@1, l_extendedprice@2, l_discount@3], metrics=[output_rows=12635, elapsed_compute=, output_bytes=B, selectivity=2.1% (12635/600572)] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], file_type=parquet, predicate=(l_quantity@1 >= Some(100),15,2 AND l_quantity@1 <= Some(1100),15,2 OR l_quantity@1 >= Some(1000),15,2 AND l_quantity@1 <= Some(2000),15,2 OR l_quantity@1 >= Some(2000),15,2 AND l_quantity@1 <= Some(3000),15,2) AND (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON AND DynamicFilter [ l_partkey@0 >= 55 AND l_partkey@0 <= 19916 ], pruning_predicate=(l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(100),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(1100),15,2 OR l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(1000),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(2000),15,2 OR l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(2000),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(3000),15,2) AND (l_shipmode_null_count@6 != row_count@2 AND l_shipmode_min@4 <= AIR AND AIR <= l_shipmode_max@5 OR l_shipmode_null_count@6 != row_count@2 AND l_shipmode_min@4 <= AIR REG AND AIR REG <= l_shipmode_max@5) AND l_shipinstruct_null_count@9 != row_count@2 AND l_shipinstruct_min@7 <= DELIVER IN PERSON AND DELIVER IN PERSON <= l_shipinstruct_max@8 AND l_partkey_null_count@11 != row_count@2 AND l_partkey_max@10 >= 55 AND l_partkey_null_count@11 != row_count@2 AND l_partkey_min@12 <= 19916, required_guarantees=[l_shipinstruct in (DELIVER IN PERSON), l_shipmode in (AIR, AIR REG)], metrics=[output_rows=600572, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=42, elapsed_compute=] │ FilterExec: (p_brand@1 = Brand#12 AND p_container@3 IN ([SM CASE, SM BOX, SM PACK, SM PKG]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([MED BAG, MED BOX, MED PKG, MED PACK]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([LG CASE, LG BOX, LG PACK, LG PKG]) AND p_size@2 <= 15) AND p_size@2 >= 1, metrics=[output_rows=42, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_brand, p_size, p_container], file_type=parquet, predicate=(p_brand@1 = Brand#12 AND p_container@3 IN ([SM CASE, SM BOX, SM PACK, SM PKG]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([MED BAG, MED BOX, MED PKG, MED PACK]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([LG CASE, LG BOX, LG PACK, LG PKG]) AND p_size@2 <= 15) AND p_size@2 >= 1, pruning_predicate=(p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#12 AND Brand#12 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM CASE AND SM CASE <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM BOX AND SM BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM PACK AND SM PACK <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM PKG AND SM PKG <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 5 OR p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BAG AND MED BAG <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED PKG AND MED PKG <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED PACK AND MED PACK <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 10 OR p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#34 AND Brand#34 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG CASE AND LG CASE <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG BOX AND LG BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG PACK AND LG PACK <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG PKG AND LG PKG <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 15) AND p_size_null_count@8 != row_count@3 AND p_size_max@9 >= 1, required_guarantees=[p_brand in (Brand#12, Brand#23, Brand#34), p_container in (LG BOX, LG CASE, LG PACK, LG PKG, MED BAG, MED BOX, MED PACK, MED PKG, SM BOX, SM CASE, SM PACK, SM PKG)], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/13.parquet], ...]}, projection=[p_partkey, p_brand, p_size, p_container], file_type=parquet, predicate=(p_brand@1 = Brand#12 AND p_container@3 IN ([SM CASE, SM BOX, SM PACK, SM PKG]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([MED BAG, MED BOX, MED PKG, MED PACK]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([LG CASE, LG BOX, LG PACK, LG PKG]) AND p_size@2 <= 15) AND p_size@2 >= 1, pruning_predicate=(p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#12 AND Brand#12 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM CASE AND SM CASE <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM BOX AND SM BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM PACK AND SM PACK <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM PKG AND SM PKG <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 5 OR p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BAG AND MED BAG <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED PKG AND MED PKG <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED PACK AND MED PACK <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 10 OR p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#34 AND Brand#34 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG CASE AND LG CASE <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG BOX AND LG BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG PACK AND LG PACK <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG PKG AND LG PKG <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 15) AND p_size_null_count@8 != row_count@3 AND p_size_max@9 >= 1, required_guarantees=[p_brand in (Brand#12, Brand#23, Brand#34), p_container in (LG BOX, LG CASE, LG PACK, LG PKG, MED BAG, MED BOX, MED PACK, MED PKG, SM BOX, SM CASE, SM PACK, SM PKG)], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -1012,8 +934,8 @@ mod tests { │ CoalescePartitionsExec, metrics=[output_rows=592, elapsed_compute=, output_bytes=B] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=592, elapsed_compute=, output_bytes=B] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[s_suppkey@1, s_name@2, s_address@3], metrics=[output_rows=592, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=, output_bytes=] - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] + │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=, output_bytes=B] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_nationkey], file_type=parquet, predicate=DynamicFilter [ s_nationkey@3 >= 3 AND s_nationkey@3 <= 3 ], pruning_predicate=s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 3 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 3, required_guarantees=[], metrics=[output_rows=1000, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=521, elapsed_compute=, output_bytes=B] │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ps_partkey@0, l_partkey@1), (ps_suppkey@1, l_suppkey@2)], filter=CAST(ps_availqty@0 AS Float64) > Float64(0.5) * sum(lineitem.l_quantity)@1, projection=[ps_suppkey@1], metrics=[output_rows=521, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] @@ -1021,32 +943,32 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=760, elapsed_compute=, output_bytes=B] │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(p_partkey@0, ps_partkey@0)], metrics=[output_rows=760, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] │ CoalescePartitionsExec, metrics=[output_rows=190, elapsed_compute=, output_bytes=B] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/partsupp/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/partsupp/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_availqty], file_type=parquet, metrics=[output_rows=80000, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] │ ProjectionExec: expr=[0.5 * CAST(sum(lineitem.l_quantity)@2 AS Float64) as Float64(0.5) * sum(lineitem.l_quantity), l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], metrics=[output_rows=54539, elapsed_compute=, output_bytes=B] │ AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=54539, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=4, metrics=[] └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] │ FilterExec: n_name@1 = CANADA, projection=[n_nationkey@0], metrics=[output_rows=16, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = CANADA, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= CANADA AND CANADA <= n_name_max@1, required_guarantees=[n_name in (CANADA)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = CANADA, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= CANADA AND CANADA <= n_name_max@1, required_guarantees=[n_name in (CANADA)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=190, elapsed_compute=] │ FilterExec: p_name@1 LIKE forest%, projection=[p_partkey@0], metrics=[output_rows=190, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet, /testdata/tpch/explain_analyze_sf0.1/part/10.parquet, /testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet, /testdata/tpch/explain_analyze_sf0.1/part/13.parquet, /testdata/tpch/explain_analyze_sf0.1/part/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/15.parquet, /testdata/tpch/explain_analyze_sf0.1/part/16.parquet, /testdata/tpch/explain_analyze_sf0.1/part/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/3.parquet, /testdata/tpch/explain_analyze_sf0.1/part/4.parquet, /testdata/tpch/explain_analyze_sf0.1/part/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/6.parquet, /testdata/tpch/explain_analyze_sf0.1/part/7.parquet, /testdata/tpch/explain_analyze_sf0.1/part/8.parquet], ...]}, projection=[p_partkey, p_name], file_type=parquet, predicate=p_name@1 LIKE forest%, pruning_predicate=p_name_null_count@2 != row_count@3 AND p_name_min@0 <= foresu AND forest <= p_name_max@1, required_guarantees=[], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/part/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/part/13.parquet], ...]}, projection=[p_partkey, p_name], file_type=parquet, predicate=p_name@1 LIKE forest%, pruning_predicate=p_name_null_count@2 != row_count@3 AND p_name_min@0 <= foresu AND forest <= p_name_max@1, required_guarantees=[], metrics=[output_rows=20000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=83515, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([l_partkey@0, l_suppkey@1], 6), input_partitions=2, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=83515, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=88801, elapsed_compute=] + │ RepartitionExec: partitioning=Hash([l_partkey@0, l_suppkey@1], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)], metrics=[output_rows=88801, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=92040, elapsed_compute=] │ FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01, projection=[l_partkey@0, l_suppkey@1, l_quantity@2], metrics=[output_rows=92040, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND DynamicFilter [ l_partkey@0 >= 5 AND l_partkey@0 <= 19852 AND l_suppkey@1 >= 4 AND l_suppkey@1 <= 1000 ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01 AND l_partkey_null_count@5 != row_count@2 AND l_partkey_max@4 >= 5 AND l_partkey_null_count@5 != row_count@2 AND l_partkey_min@6 <= 19852 AND l_suppkey_null_count@8 != row_count@2 AND l_suppkey_max@7 >= 4 AND l_suppkey_null_count@8 != row_count@2 AND l_suppkey_min@9 <= 1000, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:..], ...]}, projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND DynamicFilter [ l_partkey@0 >= 5 AND l_partkey@0 <= 19852 AND l_suppkey@1 >= 4 AND l_suppkey@1 <= 1000 ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01 AND l_partkey_null_count@5 != row_count@2 AND l_partkey_max@4 >= 5 AND l_partkey_null_count@5 != row_count@2 AND l_partkey_min@6 <= 19852 AND l_suppkey_null_count@8 != row_count@2 AND l_suppkey_max@7 >= 4 AND l_suppkey_null_count@8 != row_count@2 AND l_suppkey_min@9 <= 1000, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── "); Ok(()) @@ -1058,56 +980,50 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [numwait@1 DESC, s_name@0 ASC NULLS LAST], metrics=[output_rows=47, elapsed_compute=, output_bytes=B] - │ [Stage 4] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] + │ SortExec: expr=[numwait@1 DESC, s_name@0 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=47, elapsed_compute=, output_bytes=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] + │ ProjectionExec: expr=[s_name@0 as s_name, count(Int64(1))@1 as numwait], metrics=[output_rows=47, elapsed_compute=, output_bytes=B] + │ AggregateExec: mode=FinalPartitioned, gby=[s_name@0 as s_name], aggr=[count(Int64(1))], metrics=[output_rows=47, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=47, elapsed_compute=, output_bytes=B] + │ RepartitionExec: partitioning=Hash([s_name@0], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[s_name@0 as s_name], aggr=[count(Int64(1))], metrics=[output_rows=47, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=, reduction_factor=0.63% (47/7440)] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=7440, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0, projection=[s_name@0], metrics=[output_rows=7440, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=132720, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=132720, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0, metrics=[output_rows=132720, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=137440, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=137440, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@1)], projection=[s_name@1, l_orderkey@3, l_suppkey@4], metrics=[output_rows=137440, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=, output_bytes=B] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=182902, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@0, l_orderkey@2)], projection=[s_name@1, s_nationkey@2, l_orderkey@3, l_suppkey@4], metrics=[output_rows=182902, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=72884, elapsed_compute=, output_bytes=B] + │ [Stage 2] => NetworkCoalesceExec: output_partitions=24, input_tasks=4, metrics=[] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=379809, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_name@1, s_nationkey@2, l_orderkey@3, l_suppkey@4], metrics=[output_rows=379809, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=, output_bytes=B] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_nationkey], file_type=parquet, predicate=DynamicFilter [ s_nationkey@2 >= 20 AND s_nationkey@2 <= 20 ], pruning_predicate=s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 20 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 20, required_guarantees=[], metrics=[output_rows=1000, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=379809, elapsed_compute=, output_bytes=B] + │ FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1], metrics=[output_rows=379809, elapsed_compute=, output_bytes=B, selectivity=63% (379809/600572)] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@3 > l_commitdate@2 AND DynamicFilter [ l_suppkey@1 >= 1 AND l_suppkey@1 <= 1000 ] AND DynamicFilter [ l_orderkey@0 >= 3 AND l_orderkey@0 <= 599943 ], pruning_predicate=l_suppkey_null_count@1 != row_count@2 AND l_suppkey_max@0 >= 1 AND l_suppkey_null_count@1 != row_count@2 AND l_suppkey_min@3 <= 1000 AND l_orderkey_null_count@5 != row_count@2 AND l_orderkey_max@4 >= 3 AND l_orderkey_null_count@5 != row_count@2 AND l_orderkey_min@6 <= 599943, required_guarantees=[], metrics=[output_rows=600572, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey], file_type=parquet, metrics=[output_rows=600572, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=379809, elapsed_compute=, output_bytes=B] + │ FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1], metrics=[output_rows=379809, elapsed_compute=, output_bytes=B, selectivity=63% (379809/600572)] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@3 > l_commitdate@2, metrics=[output_rows=600572, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ SortExec: expr=[numwait@1 DESC, s_name@0 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=47, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] - │ ProjectionExec: expr=[s_name@0 as s_name, count(Int64(1))@1 as numwait], metrics=[output_rows=47, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[s_name@0 as s_name], aggr=[count(Int64(1))], metrics=[output_rows=47, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] + │ FilterExec: n_name@1 = SAUDI ARABIA, projection=[n_nationkey@0], metrics=[output_rows=16, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = SAUDI ARABIA, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= SAUDI ARABIA AND SAUDI ARABIA <= n_name_max@1, required_guarantees=[n_name in (SAUDI ARABIA)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] t3:[p18..p23] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=72884, elapsed_compute=] + │ FilterExec: o_orderstatus@1 = F, projection=[o_orderkey@0], metrics=[output_rows=72884, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,p4,p5,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4,__,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3,p4] , metrics=[] + │ DataSourceExec: file_groups={21 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:..], ...]}, projection=[o_orderkey, o_orderstatus], file_type=parquet, predicate=o_orderstatus@1 = F, pruning_predicate=o_orderstatus_null_count@2 != row_count@3 AND o_orderstatus_min@0 <= F AND F <= o_orderstatus_max@1, required_guarantees=[o_orderstatus in (F)], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=47, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([s_name@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[s_name@0 as s_name], aggr=[count(Int64(1))], metrics=[output_rows=47, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=7440, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0, projection=[s_name@0], metrics=[output_rows=7440, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=132720, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=132720, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0, metrics=[output_rows=132720, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=137440, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=137440, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@1)], projection=[s_name@1, l_orderkey@3, l_suppkey@4], metrics=[output_rows=137440, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=] - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=182902, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@0, l_orderkey@2)], projection=[s_name@1, s_nationkey@2, l_orderkey@3, l_suppkey@4], metrics=[output_rows=182902, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=72884, elapsed_compute=] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=379809, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_name@1, s_nationkey@2, l_orderkey@3, l_suppkey@4], metrics=[output_rows=379809, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=1000, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/supplier/1.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/10.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/12.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/13.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/15.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/16.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/3.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/4.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/supplier/6.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/7.parquet, /testdata/tpch/explain_analyze_sf0.1/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ], metrics=[output_rows=1000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=379809, elapsed_compute=] - │ FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1], metrics=[output_rows=379809, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@3 > l_commitdate@2 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey], file_type=parquet, metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=379809, elapsed_compute=] - │ FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1], metrics=[output_rows=379809, elapsed_compute=] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/lineitem/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/lineitem/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@3 > l_commitdate@2, metrics=[output_rows=600572, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=16, elapsed_compute=] - │ FilterExec: n_name@1 = SAUDI ARABIA, projection=[n_nationkey@0], metrics=[output_rows=16, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/nation/1.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/10.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/12.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/13.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/15.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/16.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/3.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/4.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/nation/6.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/7.parquet, /testdata/tpch/explain_analyze_sf0.1/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = SAUDI ARABIA, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= SAUDI ARABIA AND SAUDI ARABIA <= n_name_max@1, required_guarantees=[n_name in (SAUDI ARABIA)], metrics=[output_rows=400, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=72884, elapsed_compute=] - │ FilterExec: o_orderstatus@1 = F, projection=[o_orderkey@0], metrics=[output_rows=72884, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_orderstatus], file_type=parquet, predicate=o_orderstatus@1 = F, pruning_predicate=o_orderstatus_null_count@2 != row_count@3 AND o_orderstatus_min@0 <= F AND F <= o_orderstatus_max@1, required_guarantees=[o_orderstatus in (F)], metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── "); Ok(()) } @@ -1118,43 +1034,37 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [cntrycode@0 ASC NULLS LAST], metrics=[output_rows=7, elapsed_compute=, output_bytes=] - │ [Stage 4] => NetworkCoalesceExec: output_partitions=18, input_tasks=3, metrics=[] + │ SortExec: expr=[cntrycode@0 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=7, elapsed_compute=, output_bytes=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] + │ ProjectionExec: expr=[cntrycode@0 as cntrycode, count(Int64(1))@1 as numcust, sum(custsale.c_acctbal)@2 as totacctbal], metrics=[output_rows=7, elapsed_compute=, output_bytes=B] + │ AggregateExec: mode=FinalPartitioned, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)], metrics=[output_rows=7, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=7, elapsed_compute=, output_bytes=B] + │ RepartitionExec: partitioning=Hash([cntrycode@0], 6), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] + │ AggregateExec: mode=Partial, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)], metrics=[output_rows=7, elapsed_compute=, output_bytes=B, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=, reduction_factor=1.1% (7/641)] + │ ProjectionExec: expr=[substr(c_phone@1, 1, 2) as cntrycode, c_acctbal@2 as c_acctbal], metrics=[output_rows=641, elapsed_compute=, output_bytes=B] + │ NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 > avg(customer.c_acctbal)@0, projection=[avg(customer.c_acctbal)@0, c_phone@1, c_acctbal@2], metrics=[output_rows=641, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=, selectivity=47% (641/1360)] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(customer.c_acctbal)], metrics=[output_rows=1, elapsed_compute=, output_bytes=] + │ CoalescePartitionsExec, metrics=[output_rows=16, elapsed_compute=, output_bytes=B] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] + │ ProjectionExec: expr=[c_phone@0 as c_phone, c_acctbal@1 as c_acctbal, CAST(c_acctbal@1 AS Decimal128(19, 6)) as join_proj_push_down_1], metrics=[output_rows=1360, elapsed_compute=, output_bytes=B] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1360, elapsed_compute=, output_bytes=B] + │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(c_custkey@0, o_custkey@0)], projection=[c_phone@1, c_acctbal@2], metrics=[output_rows=1360, elapsed_compute=, output_bytes=B, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] + │ CoalescePartitionsExec, metrics=[output_rows=4115, elapsed_compute=, output_bytes=B] + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4, metrics=[] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_custkey], file_type=parquet, metrics=[output_rows=150000, elapsed_compute=, output_bytes=B, files_ranges_pruned_statistics= total → X matched, row_groups_pruned_statistics= total → X matched, row_groups_pruned_bloom_filter= total → X matched, page_index_rows_pruned= total → X matched, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ SortExec: expr=[cntrycode@0 ASC NULLS LAST], preserve_partitioning=[true], metrics=[output_rows=7, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, batches_split=] - │ ProjectionExec: expr=[cntrycode@0 as cntrycode, count(Int64(1))@1 as numcust, sum(custsale.c_acctbal)@2 as totacctbal], metrics=[output_rows=7, elapsed_compute=] - │ AggregateExec: mode=FinalPartitioned, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)], metrics=[output_rows=7, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1, metrics=[] + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(customer.c_acctbal)], metrics=[output_rows=16, elapsed_compute=] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=3741, elapsed_compute=] + │ FilterExec: c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]), projection=[c_acctbal@1], metrics=[output_rows=3741, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/13.parquet], ...]}, projection=[c_phone, c_acctbal], file_type=parquet, predicate=c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]), pruning_predicate=c_acctbal_null_count@1 != row_count@2 AND c_acctbal_max@0 > Some(0),15,2, required_guarantees=[], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=4115, elapsed_compute=] + │ FilterExec: substr(c_phone@1, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]), metrics=[output_rows=4115, elapsed_compute=] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] , metrics=[] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/10.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/13.parquet], ...]}, projection=[c_custkey, c_phone, c_acctbal], file_type=parquet, predicate=substr(c_phone@1, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]), metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=7, elapsed_compute=] - │ RepartitionExec: partitioning=Hash([cntrycode@0], 18), input_partitions=6, metrics=[spill_count=, spilled_bytes=, spilled_rows=, fetch_time=, repartition_time=, send_time=] - │ AggregateExec: mode=Partial, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)], metrics=[output_rows=7, elapsed_compute=, spill_count=, spilled_bytes=, spilled_rows=, skipped_aggregation_rows=, peak_mem_used=, aggregate_arguments_time=, aggregation_time=, emitting_time=, time_calculating_group_ids=] - │ ProjectionExec: expr=[substr(c_phone@1, 1, 2) as cntrycode, c_acctbal@2 as c_acctbal], metrics=[output_rows=641, elapsed_compute=] - │ NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 > avg(customer.c_acctbal)@0, projection=[avg(customer.c_acctbal)@0, c_phone@1, c_acctbal@2], metrics=[output_rows=641, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ AggregateExec: mode=Final, gby=[], aggr=[avg(customer.c_acctbal)], metrics=[output_rows=1, elapsed_compute=] - │ CoalescePartitionsExec, metrics=[output_rows=8, elapsed_compute=] - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ ProjectionExec: expr=[c_phone@0 as c_phone, c_acctbal@1 as c_acctbal, CAST(c_acctbal@1 AS Decimal128(19, 6)) as join_proj_push_down_1], metrics=[output_rows=1360, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=1360, elapsed_compute=] - │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(c_custkey@0, o_custkey@0)], projection=[c_phone@1, c_acctbal@2], metrics=[output_rows=1360, elapsed_compute=, build_input_batches=, build_input_rows=, input_batches=, input_rows=, output_batches=, build_mem_used=, build_time=, join_time=] - │ CoalescePartitionsExec, metrics=[output_rows=4115, elapsed_compute=] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4, metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/orders/1.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/10.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/11.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/12.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/13.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/14.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/15.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/16.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/2.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/3.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:..], [/testdata/tpch/explain_analyze_sf0.1/orders/4.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/5.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/6.parquet:.., /testdata/tpch/explain_analyze_sf0.1/orders/7.parquet:..], ...]}, projection=[o_custkey], file_type=parquet, metrics=[output_rows=150000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ AggregateExec: mode=Partial, gby=[], aggr=[avg(customer.c_acctbal)], metrics=[output_rows=8, elapsed_compute=] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=3741, elapsed_compute=] - │ FilterExec: c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]), projection=[c_acctbal@1], metrics=[output_rows=3741, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_phone, c_acctbal], file_type=parquet, predicate=c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]), pruning_predicate=c_acctbal_null_count@1 != row_count@2 AND c_acctbal_max@0 > Some(0),15,2, required_guarantees=[], metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192, metrics=[output_rows=4115, elapsed_compute=] - │ FilterExec: substr(c_phone@1, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]), metrics=[output_rows=4115, elapsed_compute=] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] , metrics=[] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/explain_analyze_sf0.1/customer/1.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/10.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/11.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/12.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/13.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/14.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/15.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/16.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/2.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/3.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/4.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/5.parquet], [/testdata/tpch/explain_analyze_sf0.1/customer/6.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/7.parquet, /testdata/tpch/explain_analyze_sf0.1/customer/8.parquet], ...]}, projection=[c_custkey, c_phone, c_acctbal], file_type=parquet, predicate=substr(c_phone@1, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]), metrics=[output_rows=15000, elapsed_compute=, batches_split=, bytes_scanned=, file_open_errors=, file_scan_errors=, num_predicate_creation_errors=, predicate_cache_inner_records=0, predicate_cache_records=0, predicate_evaluation_errors=, pushdown_rows_matched=, pushdown_rows_pruned=, bloom_filter_eval_time=, metadata_load_time=, page_index_eval_time=, row_pushdown_eval_time=, statistics_eval_time=, time_elapsed_opening=, time_elapsed_processing=, time_elapsed_scanning_total=, time_elapsed_scanning_until_data=] - └────────────────────────────────────────────────── "); Ok(()) } diff --git a/tests/tpch_plans_test.rs b/tests/tpch_plans_test.rs index 667274f1..3dcee13d 100644 --- a/tests/tpch_plans_test.rs +++ b/tests/tpch_plans_test.rs @@ -29,13 +29,13 @@ mod tests { └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p11] t1:[p0..p11] t2:[p0..p11] t3:[p0..p11] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([l_returnflag@0, l_linestatus@1], 12), input_partitions=2 + │ RepartitionExec: partitioning=Hash([l_returnflag@0, l_linestatus@1], 12), input_partitions=4 │ AggregateExec: mode=Partial, gby=[l_returnflag@5 as l_returnflag, l_linestatus@6 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))] │ ProjectionExec: expr=[l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as __common_expr_1, l_quantity@0 as l_quantity, l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, l_tax@3 as l_tax, l_returnflag@4 as l_returnflag, l_linestatus@5 as l_linestatus] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: l_shipdate@6 <= 1998-09-02, projection=[l_quantity@0, l_extendedprice@1, l_discount@2, l_tax@3, l_returnflag@4, l_linestatus@5] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate], file_type=parquet, predicate=l_shipdate@6 <= 1998-09-02, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@0 <= 1998-09-02, required_guarantees=[] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate], file_type=parquet, predicate=l_shipdate@6 <= 1998-09-02, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@0 <= 1998-09-02, required_guarantees=[] └────────────────────────────────────────────────── "); Ok(()) @@ -47,84 +47,72 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST] - │ [Stage 7] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 + │ SortExec: expr=[s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[s_acctbal@5 as s_acctbal, s_name@2 as s_name, n_name@7 as n_name, p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_address@3 as s_address, s_phone@4 as s_phone, s_comment@6 as s_comment] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@0, ps_partkey@1), (ps_supplycost@7, min(partsupp.ps_supplycost)@0)], projection=[p_partkey@0, p_mfgr@1, s_name@2, s_address@3, s_phone@4, s_acctbal@5, s_comment@6, n_name@8] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([p_partkey@0, ps_supplycost@7], 6), input_partitions=6 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@9)], projection=[p_partkey@1, p_mfgr@2, s_name@3, s_address@4, s_phone@5, s_acctbal@6, s_comment@7, ps_supplycost@8, n_name@9] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[p_partkey@2 as p_partkey, p_mfgr@3 as p_mfgr, s_name@4 as s_name, s_address@5 as s_address, s_phone@6 as s_phone, s_acctbal@7 as s_acctbal, s_comment@8 as s_comment, ps_supplycost@9 as ps_supplycost, n_name@0 as n_name, n_regionkey@1 as n_regionkey] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@4)], projection=[n_name@1, n_regionkey@2, p_partkey@3, p_mfgr@4, s_name@5, s_address@6, s_phone@8, s_acctbal@9, s_comment@10, ps_supplycost@11] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[p_partkey@6 as p_partkey, p_mfgr@7 as p_mfgr, s_name@0 as s_name, s_address@1 as s_address, s_nationkey@2 as s_nationkey, s_phone@3 as s_phone, s_acctbal@4 as s_acctbal, s_comment@5 as s_comment, ps_supplycost@8 as ps_supplycost] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@2)], projection=[s_name@1, s_address@2, s_nationkey@3, s_phone@4, s_acctbal@5, s_comment@6, p_partkey@7, p_mfgr@8, ps_supplycost@10] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, ps_partkey@0)], projection=[p_partkey@0, p_mfgr@1, ps_suppkey@3, ps_supplycost@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ps_partkey@1, min(partsupp.ps_supplycost)@0], 6), input_partitions=6 + │ ProjectionExec: expr=[min(partsupp.ps_supplycost)@1 as min(partsupp.ps_supplycost), ps_partkey@0 as ps_partkey] + │ AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ps_partkey@0], 6), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@2)], projection=[ps_partkey@1, ps_supplycost@2] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_supplycost@2 as ps_supplycost, n_regionkey@0 as n_regionkey] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_regionkey@1, ps_partkey@2, ps_supplycost@3] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_supplycost@2 as ps_supplycost, s_nationkey@0 as s_nationkey] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@1)], projection=[s_nationkey@1, ps_partkey@2, ps_supplycost@4] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ SortExec: expr=[s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[s_acctbal@5 as s_acctbal, s_name@2 as s_name, n_name@7 as n_name, p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_address@3 as s_address, s_phone@4 as s_phone, s_comment@6 as s_comment] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@0, ps_partkey@1), (ps_supplycost@7, min(partsupp.ps_supplycost)@0)], projection=[p_partkey@0, p_mfgr@1, s_name@2, s_address@3, s_phone@4, s_acctbal@5, s_comment@6, n_name@8] - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=3 + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/region/1.parquet], [/testdata/tpch/plan_sf0.02/region/10.parquet], [/testdata/tpch/plan_sf0.02/region/11.parquet], [/testdata/tpch/plan_sf0.02/region/12.parquet], [/testdata/tpch/plan_sf0.02/region/13.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = EUROPE, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= EUROPE AND EUROPE <= r_name_max@1, required_guarantees=[r_name in (EUROPE)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: p_size@3 = 15 AND p_type@2 LIKE %BRASS, projection=[p_partkey@0, p_mfgr@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet], [/testdata/tpch/plan_sf0.02/part/10.parquet], [/testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet], [/testdata/tpch/plan_sf0.02/part/13.parquet], ...]}, projection=[p_partkey, p_mfgr, p_type, p_size], file_type=parquet, predicate=p_size@3 = 15 AND p_type@2 LIKE %BRASS, pruning_predicate=p_size_null_count@2 != row_count@3 AND p_size_min@0 <= 15 AND 15 <= p_size_max@1, required_guarantees=[p_size in (15)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/region/1.parquet], [/testdata/tpch/plan_sf0.02/region/10.parquet], [/testdata/tpch/plan_sf0.02/region/11.parquet], [/testdata/tpch/plan_sf0.02/region/12.parquet], [/testdata/tpch/plan_sf0.02/region/13.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = EUROPE, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= EUROPE AND EUROPE <= r_name_max@1, required_guarantees=[r_name in (EUROPE)] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p23] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([p_partkey@0, ps_supplycost@7], 24), input_partitions=6 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@9)], projection=[p_partkey@1, p_mfgr@2, s_name@3, s_address@4, s_phone@5, s_acctbal@6, s_comment@7, ps_supplycost@8, n_name@9] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ ProjectionExec: expr=[p_partkey@2 as p_partkey, p_mfgr@3 as p_mfgr, s_name@4 as s_name, s_address@5 as s_address, s_phone@6 as s_phone, s_acctbal@7 as s_acctbal, s_comment@8 as s_comment, ps_supplycost@9 as ps_supplycost, n_name@0 as n_name, n_regionkey@1 as n_regionkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@4)], projection=[n_name@1, n_regionkey@2, p_partkey@3, p_mfgr@4, s_name@5, s_address@6, s_phone@8, s_acctbal@9, s_comment@10, ps_supplycost@11] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ ProjectionExec: expr=[p_partkey@6 as p_partkey, p_mfgr@7 as p_mfgr, s_name@0 as s_name, s_address@1 as s_address, s_nationkey@2 as s_nationkey, s_phone@3 as s_phone, s_acctbal@4 as s_acctbal, s_comment@5 as s_comment, ps_supplycost@8 as ps_supplycost] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@2)], projection=[s_name@1, s_address@2, s_nationkey@3, s_phone@4, s_acctbal@5, s_comment@6, p_partkey@7, p_mfgr@8, ps_supplycost@10] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment], file_type=parquet, predicate=DynamicFilter [ empty ] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, ps_partkey@0)], projection=[p_partkey@0, p_mfgr@1, ps_suppkey@3, ps_supplycost@4] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/region/1.parquet, /testdata/tpch/plan_sf0.02/region/10.parquet, /testdata/tpch/plan_sf0.02/region/11.parquet], [/testdata/tpch/plan_sf0.02/region/12.parquet, /testdata/tpch/plan_sf0.02/region/13.parquet, /testdata/tpch/plan_sf0.02/region/14.parquet], [/testdata/tpch/plan_sf0.02/region/15.parquet, /testdata/tpch/plan_sf0.02/region/16.parquet, /testdata/tpch/plan_sf0.02/region/2.parquet], [/testdata/tpch/plan_sf0.02/region/3.parquet, /testdata/tpch/plan_sf0.02/region/4.parquet, /testdata/tpch/plan_sf0.02/region/5.parquet], [/testdata/tpch/plan_sf0.02/region/6.parquet, /testdata/tpch/plan_sf0.02/region/7.parquet, /testdata/tpch/plan_sf0.02/region/8.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = EUROPE, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= EUROPE AND EUROPE <= r_name_max@1, required_guarantees=[r_name in (EUROPE)] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: p_size@3 = 15 AND p_type@2 LIKE %BRASS, projection=[p_partkey@0, p_mfgr@1] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet, /testdata/tpch/plan_sf0.02/part/10.parquet, /testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet, /testdata/tpch/plan_sf0.02/part/13.parquet, /testdata/tpch/plan_sf0.02/part/14.parquet], [/testdata/tpch/plan_sf0.02/part/15.parquet, /testdata/tpch/plan_sf0.02/part/16.parquet, /testdata/tpch/plan_sf0.02/part/2.parquet], [/testdata/tpch/plan_sf0.02/part/3.parquet, /testdata/tpch/plan_sf0.02/part/4.parquet, /testdata/tpch/plan_sf0.02/part/5.parquet], [/testdata/tpch/plan_sf0.02/part/6.parquet, /testdata/tpch/plan_sf0.02/part/7.parquet, /testdata/tpch/plan_sf0.02/part/8.parquet], ...]}, projection=[p_partkey, p_mfgr, p_type, p_size], file_type=parquet, predicate=p_size@3 = 15 AND p_type@2 LIKE %BRASS, pruning_predicate=p_size_null_count@2 != row_count@3 AND p_size_min@0 <= 15 AND 15 <= p_size_max@1, required_guarantees=[p_size in (15)] - └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ps_partkey@1, min(partsupp.ps_supplycost)@0], 24), input_partitions=6 - │ ProjectionExec: expr=[min(partsupp.ps_supplycost)@1 as min(partsupp.ps_supplycost), ps_partkey@0 as ps_partkey] - │ AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)] - │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ps_partkey@0], 18), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@2)], projection=[ps_partkey@1, ps_supplycost@2] - │ CoalescePartitionsExec - │ [Stage 4] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_supplycost@2 as ps_supplycost, n_regionkey@0 as n_regionkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_regionkey@1, ps_partkey@2, ps_supplycost@3] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_supplycost@2 as ps_supplycost, s_nationkey@0 as s_nationkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@1)], projection=[s_nationkey@1, ps_partkey@2, ps_supplycost@4] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/region/1.parquet, /testdata/tpch/plan_sf0.02/region/10.parquet, /testdata/tpch/plan_sf0.02/region/11.parquet], [/testdata/tpch/plan_sf0.02/region/12.parquet, /testdata/tpch/plan_sf0.02/region/13.parquet, /testdata/tpch/plan_sf0.02/region/14.parquet], [/testdata/tpch/plan_sf0.02/region/15.parquet, /testdata/tpch/plan_sf0.02/region/16.parquet, /testdata/tpch/plan_sf0.02/region/2.parquet], [/testdata/tpch/plan_sf0.02/region/3.parquet, /testdata/tpch/plan_sf0.02/region/4.parquet, /testdata/tpch/plan_sf0.02/region/5.parquet], [/testdata/tpch/plan_sf0.02/region/6.parquet, /testdata/tpch/plan_sf0.02/region/7.parquet, /testdata/tpch/plan_sf0.02/region/8.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = EUROPE, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= EUROPE AND EUROPE <= r_name_max@1, required_guarantees=[r_name in (EUROPE)] - └────────────────────────────────────────────────── "); Ok(()) } @@ -135,38 +123,32 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [revenue@1 DESC, o_orderdate@2 ASC NULLS LAST] - │ [Stage 3] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + │ SortExec: expr=[revenue@1 DESC, o_orderdate@2 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[l_orderkey@0 as l_orderkey, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 as revenue, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority] + │ AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([l_orderkey@0, o_orderdate@1, o_shippriority@2], 6), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[l_orderkey@2 as l_orderkey, o_orderdate@0 as o_orderdate, o_shippriority@1 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, o_shippriority@2, l_orderkey@3, l_extendedprice@4, l_discount@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_custkey@0, o_custkey@1)], projection=[o_orderkey@1, o_orderdate@3, o_shippriority@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: o_orderdate@2 < 1995-03-15 + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_custkey, o_orderdate, o_shippriority], file_type=parquet, predicate=o_orderdate@2 < 1995-03-15 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@0 < 1995-03-15, required_guarantees=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: l_shipdate@3 > 1995-03-15, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 > 1995-03-15 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 > 1995-03-15, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ SortExec: expr=[revenue@1 DESC, o_orderdate@2 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[l_orderkey@0 as l_orderkey, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 as revenue, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority] - │ AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=1 + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: c_mktsegment@1 = BUILDING, projection=[c_custkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet], [/testdata/tpch/plan_sf0.02/customer/10.parquet], [/testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet], [/testdata/tpch/plan_sf0.02/customer/13.parquet], ...]}, projection=[c_custkey, c_mktsegment], file_type=parquet, predicate=c_mktsegment@1 = BUILDING, pruning_predicate=c_mktsegment_null_count@2 != row_count@3 AND c_mktsegment_min@0 <= BUILDING AND BUILDING <= c_mktsegment_max@1, required_guarantees=[c_mktsegment in (BUILDING)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([l_orderkey@0, o_orderdate@1, o_shippriority@2], 18), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[l_orderkey@2 as l_orderkey, o_orderdate@0 as o_orderdate, o_shippriority@1 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, o_shippriority@2, l_orderkey@3, l_extendedprice@4, l_discount@5] - │ CoalescePartitionsExec - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_custkey@0, o_custkey@1)], projection=[o_orderkey@1, o_orderdate@3, o_shippriority@4] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: o_orderdate@2 < 1995-03-15 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_custkey, o_orderdate, o_shippriority], file_type=parquet, predicate=o_orderdate@2 < 1995-03-15 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@0 < 1995-03-15, required_guarantees=[] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: l_shipdate@3 > 1995-03-15, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 > 1995-03-15 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 > 1995-03-15, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: c_mktsegment@1 = BUILDING, projection=[c_custkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_mktsegment], file_type=parquet, predicate=c_mktsegment@1 = BUILDING, pruning_predicate=c_mktsegment_null_count@2 != row_count@3 AND c_mktsegment_min@0 <= BUILDING AND BUILDING <= c_mktsegment_max@1, required_guarantees=[c_mktsegment in (BUILDING)] - └────────────────────────────────────────────────── "); Ok(()) } @@ -177,32 +159,26 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [o_orderpriority@0 ASC NULLS LAST] - │ [Stage 3] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + │ SortExec: expr=[o_orderpriority@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[o_orderpriority@0 as o_orderpriority, count(Int64(1))@1 as order_count] + │ AggregateExec: mode=FinalPartitioned, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([o_orderpriority@0], 6), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(l_orderkey@0, o_orderkey@0)], projection=[o_orderpriority@1] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: o_orderdate@1 >= 1993-07-01 AND o_orderdate@1 < 1993-10-01, projection=[o_orderkey@0, o_orderpriority@2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_orderdate, o_orderpriority], file_type=parquet, predicate=o_orderdate@1 >= 1993-07-01 AND o_orderdate@1 < 1993-10-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-07-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1993-10-01, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ SortExec: expr=[o_orderpriority@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[o_orderpriority@0 as o_orderpriority, count(Int64(1))@1 as order_count] - │ AggregateExec: mode=FinalPartitioned, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))] - │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=1 + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: l_receiptdate@2 > l_commitdate@1, projection=[l_orderkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_orderkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@2 > l_commitdate@1 └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([o_orderpriority@0], 18), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(l_orderkey@0, o_orderkey@0)], projection=[o_orderpriority@1] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: o_orderdate@1 >= 1993-07-01 AND o_orderdate@1 < 1993-10-01, projection=[o_orderkey@0, o_orderpriority@2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_orderdate, o_orderpriority], file_type=parquet, predicate=o_orderdate@1 >= 1993-07-01 AND o_orderdate@1 < 1993-10-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-07-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1993-10-01, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: l_receiptdate@2 > l_commitdate@1, projection=[l_orderkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@2 > l_commitdate@1 - └────────────────────────────────────────────────── "); Ok(()) } @@ -213,55 +189,49 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [revenue@1 DESC] - │ [Stage 4] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + │ SortExec: expr=[revenue@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[n_name@0 as n_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as revenue] + │ AggregateExec: mode=FinalPartitioned, gby=[n_name@0 as n_name], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([n_name@0], 6), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[n_name@2 as n_name], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@3)], projection=[l_extendedprice@1, l_discount@2, n_name@3] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, n_name@0 as n_name, n_regionkey@1 as n_regionkey] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_name@1, n_regionkey@2, l_extendedprice@3, l_discount@4] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, s_nationkey@0 as s_nationkey] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1), (s_nationkey@1, c_nationkey@0)], projection=[s_nationkey@1, l_extendedprice@4, l_discount@5] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@1, l_orderkey@0)], projection=[c_nationkey@0, l_suppkey@3, l_extendedprice@4, l_discount@5] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[c_nationkey@1 as c_nationkey, o_orderkey@0 as o_orderkey] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_custkey@1, c_custkey@0)], projection=[o_orderkey@0, c_nationkey@3] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ SortExec: expr=[revenue@1 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[n_name@0 as n_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as revenue] - │ AggregateExec: mode=FinalPartitioned, gby=[n_name@0 as n_name], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1 + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: r_name@1 = ASIA, projection=[r_regionkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/region/1.parquet], [/testdata/tpch/plan_sf0.02/region/10.parquet], [/testdata/tpch/plan_sf0.02/region/11.parquet], [/testdata/tpch/plan_sf0.02/region/12.parquet], [/testdata/tpch/plan_sf0.02/region/13.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = ASIA, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= ASIA AND ASIA <= r_name_max@1, required_guarantees=[r_name in (ASIA)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: o_orderdate@2 >= 1994-01-01 AND o_orderdate@2 < 1995-01-01, projection=[o_orderkey@0, o_custkey@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet], [/testdata/tpch/plan_sf0.02/orders/10.parquet], [/testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet], [/testdata/tpch/plan_sf0.02/orders/13.parquet], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1994-01-01 AND o_orderdate@2 < 1995-01-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1994-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1995-01-01, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([n_name@0], 18), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[n_name@2 as n_name], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@3)], projection=[l_extendedprice@1, l_discount@2, n_name@3] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, n_name@0 as n_name, n_regionkey@1 as n_regionkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_name@1, n_regionkey@2, l_extendedprice@3, l_discount@4] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, s_nationkey@0 as s_nationkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1), (s_nationkey@1, c_nationkey@0)], projection=[s_nationkey@1, l_extendedprice@4, l_discount@5] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@1, l_orderkey@0)], projection=[c_nationkey@0, l_suppkey@3, l_extendedprice@4, l_discount@5] - │ CoalescePartitionsExec - │ ProjectionExec: expr=[c_nationkey@1 as c_nationkey, o_orderkey@0 as o_orderkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_custkey@1, c_custkey@0)], projection=[o_orderkey@0, c_nationkey@3] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: r_name@1 = ASIA, projection=[r_regionkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/region/1.parquet, /testdata/tpch/plan_sf0.02/region/10.parquet, /testdata/tpch/plan_sf0.02/region/11.parquet], [/testdata/tpch/plan_sf0.02/region/12.parquet, /testdata/tpch/plan_sf0.02/region/13.parquet, /testdata/tpch/plan_sf0.02/region/14.parquet], [/testdata/tpch/plan_sf0.02/region/15.parquet, /testdata/tpch/plan_sf0.02/region/16.parquet, /testdata/tpch/plan_sf0.02/region/2.parquet], [/testdata/tpch/plan_sf0.02/region/3.parquet, /testdata/tpch/plan_sf0.02/region/4.parquet, /testdata/tpch/plan_sf0.02/region/5.parquet], [/testdata/tpch/plan_sf0.02/region/6.parquet, /testdata/tpch/plan_sf0.02/region/7.parquet, /testdata/tpch/plan_sf0.02/region/8.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = ASIA, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= ASIA AND ASIA <= r_name_max@1, required_guarantees=[r_name in (ASIA)] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: o_orderdate@2 >= 1994-01-01 AND o_orderdate@2 < 1995-01-01, projection=[o_orderkey@0, o_custkey@1] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1994-01-01 AND o_orderdate@2 < 1995-01-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1994-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1995-01-01, required_guarantees=[] - └────────────────────────────────────────────────── "); Ok(()) } @@ -274,14 +244,14 @@ mod tests { │ ProjectionExec: expr=[sum(lineitem.l_extendedprice * lineitem.l_discount)@0 as revenue] │ AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice * lineitem.l_discount)] │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] │ AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice * lineitem.l_discount)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND l_discount@2 >= Some(5),15,2 AND l_discount@2 <= Some(7),15,2 AND l_quantity@0 < Some(2400),15,2, projection=[l_extendedprice@1, l_discount@2] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_quantity, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND l_discount@2 >= Some(5),15,2 AND l_discount@2 <= Some(7),15,2 AND l_quantity@0 < Some(2400),15,2, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01 AND l_discount_null_count@5 != row_count@2 AND l_discount_max@4 >= Some(5),15,2 AND l_discount_null_count@5 != row_count@2 AND l_discount_min@6 <= Some(7),15,2 AND l_quantity_null_count@8 != row_count@2 AND l_quantity_min@7 < Some(2400),15,2, required_guarantees=[] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_quantity, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND l_discount@2 >= Some(5),15,2 AND l_discount@2 <= Some(7),15,2 AND l_quantity@0 < Some(2400),15,2, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01 AND l_discount_null_count@5 != row_count@2 AND l_discount_max@4 >= Some(5),15,2 AND l_discount_null_count@5 != row_count@2 AND l_discount_min@6 <= Some(7),15,2 AND l_quantity_null_count@8 != row_count@2 AND l_quantity_min@7 < Some(2400),15,2, required_guarantees=[] └────────────────────────────────────────────────── "); Ok(()) @@ -293,64 +263,58 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST] - │ [Stage 5] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + │ SortExec: expr=[supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year, sum(shipping.volume)@3 as revenue] + │ AggregateExec: mode=FinalPartitioned, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([supp_nation@0, cust_nation@1, l_year@2], 6), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)] + │ ProjectionExec: expr=[n_name@4 as supp_nation, n_name@0 as cust_nation, date_part(YEAR, l_shipdate@3) as l_year, l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as volume] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@3)], filter=n_name@0 = FRANCE AND n_name@1 = GERMANY OR n_name@0 = GERMANY AND n_name@1 = FRANCE, projection=[n_name@1, l_extendedprice@2, l_discount@3, l_shipdate@4, n_name@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, l_shipdate@3 as l_shipdate, c_nationkey@4 as c_nationkey, n_name@0 as n_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@0)], projection=[n_name@1, l_extendedprice@3, l_discount@4, l_shipdate@5, c_nationkey@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[s_nationkey@1 as s_nationkey, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, l_shipdate@4 as l_shipdate, c_nationkey@0 as c_nationkey] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_custkey@0, o_custkey@4)], projection=[c_nationkey@1, s_nationkey@2, l_extendedprice@3, l_discount@4, l_shipdate@5] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_orderkey@1, o_orderkey@0)], projection=[s_nationkey@0, l_extendedprice@2, l_discount@3, l_shipdate@4, o_custkey@6] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([l_orderkey@1], 6), input_partitions=6 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_nationkey@1, l_orderkey@2, l_extendedprice@4, l_discount@5, l_shipdate@6] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: l_shipdate@4 >= 1995-01-01 AND l_shipdate@4 <= 1996-12-31 + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@4 >= 1995-01-01 AND l_shipdate@4 <= 1996-12-31 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 <= 1996-12-31, required_guarantees=[] + │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ SortExec: expr=[supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year, sum(shipping.volume)@3 as revenue] - │ AggregateExec: mode=FinalPartitioned, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)] - │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=1 + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: n_name@1 = GERMANY OR n_name@1 = FRANCE + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet], [/testdata/tpch/plan_sf0.02/nation/10.parquet], [/testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet], [/testdata/tpch/plan_sf0.02/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY OR n_name@1 = FRANCE, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1 OR n_name_null_count@2 != row_count@3 AND n_name_min@0 <= FRANCE AND FRANCE <= n_name_max@1, required_guarantees=[n_name in (FRANCE, GERMANY)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: n_name@1 = FRANCE OR n_name@1 = GERMANY + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet], [/testdata/tpch/plan_sf0.02/nation/10.parquet], [/testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet], [/testdata/tpch/plan_sf0.02/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = FRANCE OR n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= FRANCE AND FRANCE <= n_name_max@1 OR n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (FRANCE, GERMANY)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([o_orderkey@0], 6), input_partitions=4 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet], [/testdata/tpch/plan_sf0.02/orders/10.parquet], [/testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet], [/testdata/tpch/plan_sf0.02/orders/13.parquet], ...]}, projection=[o_orderkey, o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([supp_nation@0, cust_nation@1, l_year@2], 18), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)] - │ ProjectionExec: expr=[n_name@4 as supp_nation, n_name@0 as cust_nation, date_part(YEAR, l_shipdate@3) as l_year, l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as volume] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@3)], filter=n_name@0 = FRANCE AND n_name@1 = GERMANY OR n_name@0 = GERMANY AND n_name@1 = FRANCE, projection=[n_name@1, l_extendedprice@2, l_discount@3, l_shipdate@4, n_name@6] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, l_shipdate@3 as l_shipdate, c_nationkey@4 as c_nationkey, n_name@0 as n_name] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@0)], projection=[n_name@1, l_extendedprice@3, l_discount@4, l_shipdate@5, c_nationkey@6] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ ProjectionExec: expr=[s_nationkey@1 as s_nationkey, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, l_shipdate@4 as l_shipdate, c_nationkey@0 as c_nationkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_custkey@0, o_custkey@4)], projection=[c_nationkey@1, s_nationkey@2, l_extendedprice@3, l_discount@4, l_shipdate@5] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_orderkey@1, o_orderkey@0)], projection=[s_nationkey@0, l_extendedprice@2, l_discount@3, l_shipdate@4, o_custkey@6] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([l_orderkey@1], 6), input_partitions=6 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_nationkey@1, l_orderkey@2, l_extendedprice@4, l_discount@5, l_shipdate@6] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: l_shipdate@4 >= 1995-01-01 AND l_shipdate@4 <= 1996-12-31 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@4 >= 1995-01-01 AND l_shipdate@4 <= 1996-12-31 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 <= 1996-12-31, required_guarantees=[] - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: n_name@1 = GERMANY OR n_name@1 = FRANCE - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY OR n_name@1 = FRANCE, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1 OR n_name_null_count@2 != row_count@3 AND n_name_min@0 <= FRANCE AND FRANCE <= n_name_max@1, required_guarantees=[n_name in (FRANCE, GERMANY)] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: n_name@1 = FRANCE OR n_name@1 = GERMANY - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = FRANCE OR n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= FRANCE AND FRANCE <= n_name_max@1 OR n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (FRANCE, GERMANY)] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([o_orderkey@0], 6), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - └────────────────────────────────────────────────── "); Ok(()) } @@ -361,71 +325,65 @@ mod tests { assert_snapshot!(plan, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [o_year@0 ASC NULLS LAST] - │ [Stage 5] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + │ SortExec: expr=[o_year@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[o_year@0 as o_year, sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END)@1 / sum(all_nations.volume)@2 as mkt_share] + │ AggregateExec: mode=FinalPartitioned, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([o_year@0], 6), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)] + │ ProjectionExec: expr=[date_part(YEAR, o_orderdate@2) as o_year, l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as volume, n_name@3 as nation] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@3)], projection=[l_extendedprice@1, l_discount@2, o_orderdate@3, n_name@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, o_orderdate@3 as o_orderdate, n_regionkey@4 as n_regionkey, n_name@0 as n_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_name@1, l_extendedprice@2, l_discount@3, o_orderdate@5, n_regionkey@6] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet + │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, s_nationkey@3 as s_nationkey, o_orderdate@4 as o_orderdate, n_regionkey@0 as n_regionkey] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@4)], projection=[n_regionkey@1, l_extendedprice@2, l_discount@3, s_nationkey@4, o_orderdate@5] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_custkey@3, c_custkey@0)], projection=[l_extendedprice@0, l_discount@1, s_nationkey@2, o_orderdate@4, c_nationkey@6] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@4 as s_nationkey, o_custkey@0 as o_custkey, o_orderdate@1 as o_orderdate] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_custkey@1, o_orderdate@2, l_extendedprice@4, l_discount@5, s_nationkey@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@0 as s_nationkey] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_nationkey@1, l_orderkey@2, l_extendedprice@4, l_discount@5] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@1, l_suppkey@3, l_extendedprice@4, l_discount@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_partkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ SortExec: expr=[o_year@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[o_year@0 as o_year, sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END)@1 / sum(all_nations.volume)@2 as mkt_share] - │ AggregateExec: mode=FinalPartitioned, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)] - │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=1 + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: r_name@1 = AMERICA, projection=[r_regionkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/region/1.parquet], [/testdata/tpch/plan_sf0.02/region/10.parquet], [/testdata/tpch/plan_sf0.02/region/11.parquet], [/testdata/tpch/plan_sf0.02/region/12.parquet], [/testdata/tpch/plan_sf0.02/region/13.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = AMERICA, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= AMERICA AND AMERICA <= r_name_max@1, required_guarantees=[r_name in (AMERICA)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: o_orderdate@2 >= 1995-01-01 AND o_orderdate@2 <= 1996-12-31 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet], [/testdata/tpch/plan_sf0.02/orders/10.parquet], [/testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet], [/testdata/tpch/plan_sf0.02/orders/13.parquet], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1995-01-01 AND o_orderdate@2 <= 1996-12-31, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1995-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 <= 1996-12-31, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: p_type@1 = ECONOMY ANODIZED STEEL, projection=[p_partkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet], [/testdata/tpch/plan_sf0.02/part/10.parquet], [/testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet], [/testdata/tpch/plan_sf0.02/part/13.parquet], ...]}, projection=[p_partkey, p_type], file_type=parquet, predicate=p_type@1 = ECONOMY ANODIZED STEEL, pruning_predicate=p_type_null_count@2 != row_count@3 AND p_type_min@0 <= ECONOMY ANODIZED STEEL AND ECONOMY ANODIZED STEEL <= p_type_max@1, required_guarantees=[p_type in (ECONOMY ANODIZED STEEL)] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([o_year@0], 18), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)] - │ ProjectionExec: expr=[date_part(YEAR, o_orderdate@2) as o_year, l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as volume, n_name@3 as nation] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@3)], projection=[l_extendedprice@1, l_discount@2, o_orderdate@3, n_name@5] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, o_orderdate@3 as o_orderdate, n_regionkey@4 as n_regionkey, n_name@0 as n_name] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_name@1, l_extendedprice@2, l_discount@3, o_orderdate@5, n_regionkey@6] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet - │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, s_nationkey@3 as s_nationkey, o_orderdate@4 as o_orderdate, n_regionkey@0 as n_regionkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@4)], projection=[n_regionkey@1, l_extendedprice@2, l_discount@3, s_nationkey@4, o_orderdate@5] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_custkey@3, c_custkey@0)], projection=[l_extendedprice@0, l_discount@1, s_nationkey@2, o_orderdate@4, c_nationkey@6] - │ CoalescePartitionsExec - │ ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@4 as s_nationkey, o_custkey@0 as o_custkey, o_orderdate@1 as o_orderdate] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_custkey@1, o_orderdate@2, l_extendedprice@4, l_discount@5, s_nationkey@6] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@0 as s_nationkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_nationkey@1, l_orderkey@2, l_extendedprice@4, l_discount@5] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@1, l_suppkey@3, l_extendedprice@4, l_discount@5] - │ CoalescePartitionsExec - │ [Stage 3] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_partkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: r_name@1 = AMERICA, projection=[r_regionkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/region/1.parquet, /testdata/tpch/plan_sf0.02/region/10.parquet, /testdata/tpch/plan_sf0.02/region/11.parquet], [/testdata/tpch/plan_sf0.02/region/12.parquet, /testdata/tpch/plan_sf0.02/region/13.parquet, /testdata/tpch/plan_sf0.02/region/14.parquet], [/testdata/tpch/plan_sf0.02/region/15.parquet, /testdata/tpch/plan_sf0.02/region/16.parquet, /testdata/tpch/plan_sf0.02/region/2.parquet], [/testdata/tpch/plan_sf0.02/region/3.parquet, /testdata/tpch/plan_sf0.02/region/4.parquet, /testdata/tpch/plan_sf0.02/region/5.parquet], [/testdata/tpch/plan_sf0.02/region/6.parquet, /testdata/tpch/plan_sf0.02/region/7.parquet, /testdata/tpch/plan_sf0.02/region/8.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = AMERICA, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= AMERICA AND AMERICA <= r_name_max@1, required_guarantees=[r_name in (AMERICA)] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: o_orderdate@2 >= 1995-01-01 AND o_orderdate@2 <= 1996-12-31 - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1995-01-01 AND o_orderdate@2 <= 1996-12-31, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1995-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 <= 1996-12-31, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: p_type@1 = ECONOMY ANODIZED STEEL, projection=[p_partkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet, /testdata/tpch/plan_sf0.02/part/10.parquet, /testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet, /testdata/tpch/plan_sf0.02/part/13.parquet, /testdata/tpch/plan_sf0.02/part/14.parquet], [/testdata/tpch/plan_sf0.02/part/15.parquet, /testdata/tpch/plan_sf0.02/part/16.parquet, /testdata/tpch/plan_sf0.02/part/2.parquet], [/testdata/tpch/plan_sf0.02/part/3.parquet, /testdata/tpch/plan_sf0.02/part/4.parquet, /testdata/tpch/plan_sf0.02/part/5.parquet], [/testdata/tpch/plan_sf0.02/part/6.parquet, /testdata/tpch/plan_sf0.02/part/7.parquet, /testdata/tpch/plan_sf0.02/part/8.parquet], ...]}, projection=[p_partkey, p_type], file_type=parquet, predicate=p_type@1 = ECONOMY ANODIZED STEEL, pruning_predicate=p_type_null_count@2 != row_count@3 AND p_type_min@0 <= ECONOMY ANODIZED STEEL AND ECONOMY ANODIZED STEEL <= p_type_max@1, required_guarantees=[p_type in (ECONOMY ANODIZED STEEL)] - └────────────────────────────────────────────────── "#); Ok(()) } @@ -436,69 +394,63 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [nation@0 ASC NULLS LAST, o_year@1 DESC] - │ [Stage 7] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + │ SortExec: expr=[nation@0 ASC NULLS LAST, o_year@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[nation@0 as nation, o_year@1 as o_year, sum(profit.amount)@2 as sum_profit] + │ AggregateExec: mode=FinalPartitioned, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([nation@0, o_year@1], 6), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)] + │ ProjectionExec: expr=[n_name@0 as nation, date_part(YEAR, o_orderdate@5) as o_year, l_extendedprice@2 * (Some(1),20,0 - l_discount@3) - ps_supplycost@4 * l_quantity@1 as amount] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[n_name@1, l_quantity@2, l_extendedprice@3, l_discount@4, ps_supplycost@6, o_orderdate@7] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet + │ ProjectionExec: expr=[l_quantity@1 as l_quantity, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@4 as s_nationkey, ps_supplycost@5 as ps_supplycost, o_orderdate@0 as o_orderdate] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, l_quantity@3, l_extendedprice@4, l_discount@5, s_nationkey@6, ps_supplycost@7] + │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ SortExec: expr=[nation@0 ASC NULLS LAST, o_year@1 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[nation@0 as nation, o_year@1 as o_year, sum(profit.amount)@2 as sum_profit] - │ AggregateExec: mode=FinalPartitioned, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)] - │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=1 + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([o_orderkey@0], 6), input_partitions=4 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet], [/testdata/tpch/plan_sf0.02/orders/10.parquet], [/testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet], [/testdata/tpch/plan_sf0.02/orders/13.parquet], ...]}, projection=[o_orderkey, o_orderdate], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p17] + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([l_orderkey@0], 6), input_partitions=6 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_suppkey@2, ps_suppkey@1), (l_partkey@1, ps_partkey@0)], projection=[l_orderkey@0, l_quantity@3, l_extendedprice@4, l_discount@5, s_nationkey@6, ps_supplycost@9] + │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1 + │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p23] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([nation@0, o_year@1], 18), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)] - │ ProjectionExec: expr=[n_name@0 as nation, date_part(YEAR, o_orderdate@5) as o_year, l_extendedprice@2 * (Some(1),20,0 - l_discount@3) - ps_supplycost@4 * l_quantity@1 as amount] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[n_name@1, l_quantity@2, l_extendedprice@3, l_discount@4, ps_supplycost@6, o_orderdate@7] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet - │ ProjectionExec: expr=[l_quantity@1 as l_quantity, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@4 as s_nationkey, ps_supplycost@5 as ps_supplycost, o_orderdate@0 as o_orderdate] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, l_quantity@3, l_extendedprice@4, l_discount@5, s_nationkey@6, ps_supplycost@7] - │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=4 - │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + │ RepartitionExec: partitioning=Hash([l_suppkey@2, l_partkey@1], 24), input_partitions=6 + │ ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_partkey@2 as l_partkey, l_suppkey@3 as l_suppkey, l_quantity@4 as l_quantity, l_extendedprice@5 as l_extendedprice, l_discount@6 as l_discount, s_nationkey@0 as s_nationkey] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@2)], projection=[s_nationkey@1, l_orderkey@2, l_partkey@3, l_suppkey@4, l_quantity@5, l_extendedprice@6, l_discount@7] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@1, l_partkey@2, l_suppkey@3, l_quantity@4, l_extendedprice@5, l_discount@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([o_orderkey@0], 6), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_orderdate], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([l_orderkey@0], 6), input_partitions=6 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_suppkey@2, ps_suppkey@1), (l_partkey@1, ps_partkey@0)], projection=[l_orderkey@0, l_quantity@3, l_extendedprice@4, l_discount@5, s_nationkey@6, ps_supplycost@9] - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + │ FilterExec: p_name@1 LIKE %green%, projection=[p_partkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet], [/testdata/tpch/plan_sf0.02/part/10.parquet], [/testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet], [/testdata/tpch/plan_sf0.02/part/13.parquet], ...]}, projection=[p_partkey, p_name], file_type=parquet, predicate=p_name@1 LIKE %green% └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p23] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([l_suppkey@2, l_partkey@1], 24), input_partitions=6 - │ ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_partkey@2 as l_partkey, l_suppkey@3 as l_suppkey, l_quantity@4 as l_quantity, l_extendedprice@5 as l_extendedprice, l_discount@6 as l_discount, s_nationkey@0 as s_nationkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@2)], projection=[s_nationkey@1, l_orderkey@2, l_partkey@3, l_suppkey@4, l_quantity@5, l_extendedprice@6, l_discount@7] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@1, l_partkey@2, l_suppkey@3, l_quantity@4, l_extendedprice@5, l_discount@6] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: p_name@1 LIKE %green%, projection=[p_partkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet, /testdata/tpch/plan_sf0.02/part/10.parquet, /testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet, /testdata/tpch/plan_sf0.02/part/13.parquet, /testdata/tpch/plan_sf0.02/part/14.parquet], [/testdata/tpch/plan_sf0.02/part/15.parquet, /testdata/tpch/plan_sf0.02/part/16.parquet, /testdata/tpch/plan_sf0.02/part/2.parquet], [/testdata/tpch/plan_sf0.02/part/3.parquet, /testdata/tpch/plan_sf0.02/part/4.parquet, /testdata/tpch/plan_sf0.02/part/5.parquet], [/testdata/tpch/plan_sf0.02/part/6.parquet, /testdata/tpch/plan_sf0.02/part/7.parquet, /testdata/tpch/plan_sf0.02/part/8.parquet], ...]}, projection=[p_partkey, p_name], file_type=parquet, predicate=p_name@1 LIKE %green% - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ps_suppkey@1, ps_partkey@0], 24), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] - └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ps_suppkey@1, ps_partkey@0], 24), input_partitions=4 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet], [/testdata/tpch/plan_sf0.02/partsupp/10.parquet], [/testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet], [/testdata/tpch/plan_sf0.02/partsupp/13.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── "); Ok(()) } @@ -509,42 +461,36 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [revenue@2 DESC] - │ [Stage 3] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + │ SortExec: expr=[revenue@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@7 as revenue, c_acctbal@2 as c_acctbal, n_name@4 as n_name, c_address@5 as c_address, c_phone@3 as c_phone, c_comment@6 as c_comment] + │ AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@2 as c_acctbal, c_phone@3 as c_phone, n_name@4 as n_name, c_address@5 as c_address, c_comment@6 as c_comment], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_custkey@0, c_name@1, c_acctbal@2, c_phone@3, n_name@4, c_address@5, c_comment@6], 6), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@4 as c_acctbal, c_phone@3 as c_phone, n_name@8 as n_name, c_address@2 as c_address, c_comment@5 as c_comment], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ ProjectionExec: expr=[c_custkey@1 as c_custkey, c_name@2 as c_name, c_address@3 as c_address, c_phone@4 as c_phone, c_acctbal@5 as c_acctbal, c_comment@6 as c_comment, l_extendedprice@7 as l_extendedprice, l_discount@8 as l_discount, n_name@0 as n_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@3)], projection=[n_name@1, c_custkey@2, c_name@3, c_address@4, c_phone@6, c_acctbal@7, c_comment@8, l_extendedprice@9, l_discount@10] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@7, l_orderkey@0)], projection=[c_custkey@0, c_name@1, c_address@2, c_nationkey@3, c_phone@4, c_acctbal@5, c_comment@6, l_extendedprice@9, l_discount@10] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[c_custkey@1 as c_custkey, c_name@2 as c_name, c_address@3 as c_address, c_nationkey@4 as c_nationkey, c_phone@5 as c_phone, c_acctbal@6 as c_acctbal, c_comment@7 as c_comment, o_orderkey@0 as o_orderkey] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_custkey@1, c_custkey@0)], projection=[o_orderkey@0, c_custkey@2, c_name@3, c_address@4, c_nationkey@5, c_phone@6, c_acctbal@7, c_comment@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_comment], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: l_returnflag@3 = R, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_extendedprice, l_discount, l_returnflag], file_type=parquet, predicate=l_returnflag@3 = R AND DynamicFilter [ empty ], pruning_predicate=l_returnflag_null_count@2 != row_count@3 AND l_returnflag_min@0 <= R AND R <= l_returnflag_max@1, required_guarantees=[l_returnflag in (R)] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ SortExec: expr=[revenue@2 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@7 as revenue, c_acctbal@2 as c_acctbal, n_name@4 as n_name, c_address@5 as c_address, c_phone@3 as c_phone, c_comment@6 as c_comment] - │ AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@2 as c_acctbal, c_phone@3 as c_phone, n_name@4 as n_name, c_address@5 as c_address, c_comment@6 as c_comment], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=1 + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: o_orderdate@2 >= 1993-10-01 AND o_orderdate@2 < 1994-01-01, projection=[o_orderkey@0, o_custkey@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet], [/testdata/tpch/plan_sf0.02/orders/10.parquet], [/testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet], [/testdata/tpch/plan_sf0.02/orders/13.parquet], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1993-10-01 AND o_orderdate@2 < 1994-01-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-10-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1994-01-01, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([c_custkey@0, c_name@1, c_acctbal@2, c_phone@3, n_name@4, c_address@5, c_comment@6], 18), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@4 as c_acctbal, c_phone@3 as c_phone, n_name@8 as n_name, c_address@2 as c_address, c_comment@5 as c_comment], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ ProjectionExec: expr=[c_custkey@1 as c_custkey, c_name@2 as c_name, c_address@3 as c_address, c_phone@4 as c_phone, c_acctbal@5 as c_acctbal, c_comment@6 as c_comment, l_extendedprice@7 as l_extendedprice, l_discount@8 as l_discount, n_name@0 as n_name] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@3)], projection=[n_name@1, c_custkey@2, c_name@3, c_address@4, c_phone@6, c_acctbal@7, c_comment@8, l_extendedprice@9, l_discount@10] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@7, l_orderkey@0)], projection=[c_custkey@0, c_name@1, c_address@2, c_nationkey@3, c_phone@4, c_acctbal@5, c_comment@6, l_extendedprice@9, l_discount@10] - │ CoalescePartitionsExec - │ ProjectionExec: expr=[c_custkey@1 as c_custkey, c_name@2 as c_name, c_address@3 as c_address, c_nationkey@4 as c_nationkey, c_phone@5 as c_phone, c_acctbal@6 as c_acctbal, c_comment@7 as c_comment, o_orderkey@0 as o_orderkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_custkey@1, c_custkey@0)], projection=[o_orderkey@0, c_custkey@2, c_name@3, c_address@4, c_nationkey@5, c_phone@6, c_acctbal@7, c_comment@8] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_comment], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: l_returnflag@3 = R, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_extendedprice, l_discount, l_returnflag], file_type=parquet, predicate=l_returnflag@3 = R AND DynamicFilter [ empty ], pruning_predicate=l_returnflag_null_count@2 != row_count@3 AND l_returnflag_min@0 <= R AND R <= l_returnflag_max@1, required_guarantees=[l_returnflag in (R)] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: o_orderdate@2 >= 1993-10-01 AND o_orderdate@2 < 1994-01-01, projection=[o_orderkey@0, o_custkey@1] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1993-10-01 AND o_orderdate@2 < 1994-01-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-10-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1994-01-01, required_guarantees=[] - └────────────────────────────────────────────────── "); Ok(()) } @@ -565,7 +511,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[ps_availqty@1, ps_supplycost@2] │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 │ ProjectionExec: expr=[ps_availqty@1 as ps_availqty, ps_supplycost@2 as ps_supplycost, s_nationkey@0 as s_nationkey] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@0)], projection=[s_nationkey@1, ps_availqty@3, ps_supplycost@4] @@ -580,7 +526,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[ps_partkey@1, ps_availqty@2, ps_supplycost@3] │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_availqty@2 as ps_availqty, ps_supplycost@3 as ps_supplycost, s_nationkey@0 as s_nationkey] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@1)], projection=[s_nationkey@1, ps_partkey@2, ps_availqty@4, ps_supplycost@5] @@ -588,17 +534,17 @@ mod tests { │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (GERMANY)] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet], [/testdata/tpch/plan_sf0.02/nation/10.parquet], [/testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet], [/testdata/tpch/plan_sf0.02/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (GERMANY)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (GERMANY)] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet], [/testdata/tpch/plan_sf0.02/nation/10.parquet], [/testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet], [/testdata/tpch/plan_sf0.02/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (GERMANY)] └────────────────────────────────────────────────── "); Ok(()) @@ -616,9 +562,9 @@ mod tests { │ SortExec: expr=[l_shipmode@0 ASC NULLS LAST], preserve_partitioning=[true] │ ProjectionExec: expr=[l_shipmode@0 as l_shipmode, sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@1 as high_line_count, sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@2 as low_line_count] │ AggregateExec: mode=FinalPartitioned, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)] - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=3 + │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] + ┌───── Stage 3 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([l_shipmode@0], 18), input_partitions=6 │ AggregateExec: mode=Partial, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)] @@ -627,19 +573,19 @@ mod tests { │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=4 │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] + ┌───── Stage 1 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([l_orderkey@0], 18), input_partitions=2 + │ RepartitionExec: partitioning=Hash([l_orderkey@0], 24), input_partitions=4 │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: (l_shipmode@4 = MAIL OR l_shipmode@4 = SHIP) AND l_receiptdate@3 > l_commitdate@2 AND l_shipdate@1 < l_commitdate@2 AND l_receiptdate@3 >= 1994-01-01 AND l_receiptdate@3 < 1995-01-01, projection=[l_orderkey@0, l_shipmode@4] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_shipdate, l_commitdate, l_receiptdate, l_shipmode], file_type=parquet, predicate=(l_shipmode@4 = MAIL OR l_shipmode@4 = SHIP) AND l_receiptdate@3 > l_commitdate@2 AND l_shipdate@1 < l_commitdate@2 AND l_receiptdate@3 >= 1994-01-01 AND l_receiptdate@3 < 1995-01-01, pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= MAIL AND MAIL <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1) AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_max@4 >= 1994-01-01 AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_min@6 < 1995-01-01, required_guarantees=[l_shipmode in (MAIL, SHIP)] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_orderkey, l_shipdate, l_commitdate, l_receiptdate, l_shipmode], file_type=parquet, predicate=(l_shipmode@4 = MAIL OR l_shipmode@4 = SHIP) AND l_receiptdate@3 > l_commitdate@2 AND l_shipdate@1 < l_commitdate@2 AND l_receiptdate@3 >= 1994-01-01 AND l_receiptdate@3 < 1995-01-01, pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= MAIL AND MAIL <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1) AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_max@4 >= 1994-01-01 AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_min@6 < 1995-01-01, required_guarantees=[l_shipmode in (MAIL, SHIP)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] + ┌───── Stage 2 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([o_orderkey@0], 18), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_orderpriority], file_type=parquet, predicate=DynamicFilter [ empty ] + │ RepartitionExec: partitioning=Hash([o_orderkey@0], 24), input_partitions=4 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet], [/testdata/tpch/plan_sf0.02/orders/10.parquet], [/testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet], [/testdata/tpch/plan_sf0.02/orders/13.parquet], ...]}, projection=[o_orderkey, o_orderpriority], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── "#); Ok(()) @@ -651,39 +597,30 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [custdist@1 DESC, c_count@0 DESC] - │ [Stage 4] => NetworkCoalesceExec: output_partitions=12, input_tasks=2 + │ SortExec: expr=[custdist@1 DESC, c_count@0 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[c_count@0 as c_count, count(Int64(1))@1 as custdist] + │ AggregateExec: mode=FinalPartitioned, gby=[c_count@0 as c_count], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_count@0], 6), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[c_count@0 as c_count], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[count(orders.o_orderkey)@1 as c_count] + │ AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey], aggr=[count(orders.o_orderkey)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_custkey@0], 6), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey], aggr=[count(orders.o_orderkey)] + │ ProjectionExec: expr=[c_custkey@1 as c_custkey, o_orderkey@0 as o_orderkey] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(o_custkey@1, c_custkey@0)], projection=[o_orderkey@0, c_custkey@2] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] - │ SortExec: expr=[custdist@1 DESC, c_count@0 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[c_count@0 as c_count, count(Int64(1))@1 as custdist] - │ AggregateExec: mode=FinalPartitioned, gby=[c_count@0 as c_count], aggr=[count(Int64(1))] - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=3 + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: o_comment@2 NOT LIKE %special%requests%, projection=[o_orderkey@0, o_custkey@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet], [/testdata/tpch/plan_sf0.02/orders/10.parquet], [/testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet], [/testdata/tpch/plan_sf0.02/orders/13.parquet], ...]}, projection=[o_orderkey, o_custkey, o_comment], file_type=parquet, predicate=o_comment@2 NOT LIKE %special%requests% └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p11] t1:[p0..p11] t2:[p0..p11] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([c_count@0], 12), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[c_count@0 as c_count], aggr=[count(Int64(1))] - │ ProjectionExec: expr=[count(orders.o_orderkey)@1 as c_count] - │ AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey], aggr=[count(orders.o_orderkey)] - │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([c_custkey@0], 18), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey], aggr=[count(orders.o_orderkey)] - │ ProjectionExec: expr=[c_custkey@1 as c_custkey, o_orderkey@0 as o_orderkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(o_custkey@1, c_custkey@0)], projection=[o_orderkey@0, c_custkey@2] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: o_comment@2 NOT LIKE %special%requests%, projection=[o_orderkey@0, o_custkey@1] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_custkey, o_comment], file_type=parquet, predicate=o_comment@2 NOT LIKE %special%requests% - └────────────────────────────────────────────────── "); Ok(()) } @@ -736,21 +673,21 @@ mod tests { └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p11] t1:[p0..p11] t2:[p0..p11] t3:[p0..p11] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([l_suppkey@0], 12), input_partitions=2 + │ RepartitionExec: partitioning=Hash([l_suppkey@0], 12), input_partitions=4 │ AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([l_suppkey@0], 6), input_partitions=2 + │ RepartitionExec: partitioning=Hash([l_suppkey@0], 6), input_partitions=4 │ AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] └────────────────────────────────────────────────── "); Ok(()) @@ -762,48 +699,39 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST] - │ [Stage 5] => NetworkCoalesceExec: output_partitions=12, input_tasks=2 + │ SortExec: expr=[supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, count(alias1)@3 as supplier_cnt] + │ AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2], 6), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)] + │ AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, alias1@3 as alias1], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2, alias1@3], 6), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[p_brand@1 as p_brand, p_type@2 as p_type, p_size@3 as p_size, ps_suppkey@0 as alias1], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(s_suppkey@0, ps_suppkey@0)] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[ps_suppkey@3 as ps_suppkey, p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, ps_partkey@0)], projection=[p_brand@1, p_type@2, p_size@3, ps_suppkey@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_partkey, ps_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] - │ SortExec: expr=[supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, count(alias1)@3 as supplier_cnt] - │ AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)] - │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=3 + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_comment@1 LIKE %Customer%Complaints%, projection=[s_suppkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet], [/testdata/tpch/plan_sf0.02/supplier/10.parquet], [/testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet], [/testdata/tpch/plan_sf0.02/supplier/13.parquet], ...]}, projection=[s_suppkey, s_comment], file_type=parquet, predicate=s_comment@1 LIKE %Customer%Complaints% + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND p_size@3 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]) + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet], [/testdata/tpch/plan_sf0.02/part/10.parquet], [/testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet], [/testdata/tpch/plan_sf0.02/part/13.parquet], ...]}, projection=[p_partkey, p_brand, p_type, p_size], file_type=parquet, predicate=p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND p_size@3 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]), pruning_predicate=p_brand_null_count@2 != row_count@3 AND (p_brand_min@0 != Brand#45 OR Brand#45 != p_brand_max@1) AND p_type_null_count@6 != row_count@3 AND (p_type_min@4 NOT LIKE MEDIUM POLISHED% OR p_type_max@5 NOT LIKE MEDIUM POLISHED%) AND (p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 49 AND 49 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 14 AND 14 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 23 AND 23 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 45 AND 45 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 19 AND 19 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 3 AND 3 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 36 AND 36 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 9 AND 9 <= p_size_max@8), required_guarantees=[p_brand not in (Brand#45), p_size in (14, 19, 23, 3, 36, 45, 49, 9)] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p11] t1:[p0..p11] t2:[p0..p11] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2], 12), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)] - │ AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, alias1@3 as alias1], aggr=[] - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2, alias1@3], 18), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[p_brand@1 as p_brand, p_type@2 as p_type, p_size@3 as p_size, ps_suppkey@0 as alias1], aggr=[] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(s_suppkey@0, ps_suppkey@0)] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ ProjectionExec: expr=[ps_suppkey@3 as ps_suppkey, p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, ps_partkey@0)], projection=[p_brand@1, p_type@2, p_size@3, ps_suppkey@5] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_partkey, ps_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: s_comment@1 LIKE %Customer%Complaints%, projection=[s_suppkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_comment], file_type=parquet, predicate=s_comment@1 LIKE %Customer%Complaints% - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND p_size@3 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]) - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet, /testdata/tpch/plan_sf0.02/part/10.parquet, /testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet, /testdata/tpch/plan_sf0.02/part/13.parquet, /testdata/tpch/plan_sf0.02/part/14.parquet], [/testdata/tpch/plan_sf0.02/part/15.parquet, /testdata/tpch/plan_sf0.02/part/16.parquet, /testdata/tpch/plan_sf0.02/part/2.parquet], [/testdata/tpch/plan_sf0.02/part/3.parquet, /testdata/tpch/plan_sf0.02/part/4.parquet, /testdata/tpch/plan_sf0.02/part/5.parquet], [/testdata/tpch/plan_sf0.02/part/6.parquet, /testdata/tpch/plan_sf0.02/part/7.parquet, /testdata/tpch/plan_sf0.02/part/8.parquet], ...]}, projection=[p_partkey, p_brand, p_type, p_size], file_type=parquet, predicate=p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND p_size@3 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]), pruning_predicate=p_brand_null_count@2 != row_count@3 AND (p_brand_min@0 != Brand#45 OR Brand#45 != p_brand_max@1) AND p_type_null_count@6 != row_count@3 AND (p_type_min@4 NOT LIKE MEDIUM POLISHED% OR p_type_max@5 NOT LIKE MEDIUM POLISHED%) AND (p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 49 AND 49 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 14 AND 14 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 23 AND 23 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 45 AND 45 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 19 AND 19 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 3 AND 3 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 36 AND 36 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 9 AND 9 <= p_size_max@8), required_guarantees=[p_brand not in (Brand#45), p_size in (14, 19, 23, 3, 36, 45, 49, 9)] - └────────────────────────────────────────────────── "); Ok(()) } @@ -824,24 +752,24 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@0)], projection=[p_partkey@0, l_quantity@2, l_extendedprice@3] │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_partkey, l_quantity, l_extendedprice], file_type=parquet, predicate=DynamicFilter [ empty ] │ ProjectionExec: expr=[CAST(0.2 * CAST(avg(lineitem.l_quantity)@1 AS Float64) AS Decimal128(30, 15)) as Float64(0.2) * avg(lineitem.l_quantity), l_partkey@0 as l_partkey] │ AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)] │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: p_brand@1 = Brand#23 AND p_container@2 = MED BOX, projection=[p_partkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet, /testdata/tpch/plan_sf0.02/part/10.parquet, /testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet, /testdata/tpch/plan_sf0.02/part/13.parquet, /testdata/tpch/plan_sf0.02/part/14.parquet], [/testdata/tpch/plan_sf0.02/part/15.parquet, /testdata/tpch/plan_sf0.02/part/16.parquet, /testdata/tpch/plan_sf0.02/part/2.parquet], [/testdata/tpch/plan_sf0.02/part/3.parquet, /testdata/tpch/plan_sf0.02/part/4.parquet, /testdata/tpch/plan_sf0.02/part/5.parquet], [/testdata/tpch/plan_sf0.02/part/6.parquet, /testdata/tpch/plan_sf0.02/part/7.parquet, /testdata/tpch/plan_sf0.02/part/8.parquet], ...]}, projection=[p_partkey, p_brand, p_container], file_type=parquet, predicate=p_brand@1 = Brand#23 AND p_container@2 = MED BOX, pruning_predicate=p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5, required_guarantees=[p_brand in (Brand#23), p_container in (MED BOX)] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet], [/testdata/tpch/plan_sf0.02/part/10.parquet], [/testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet], [/testdata/tpch/plan_sf0.02/part/13.parquet], ...]}, projection=[p_partkey, p_brand, p_container], file_type=parquet, predicate=p_brand@1 = Brand#23 AND p_container@2 = MED BOX, pruning_predicate=p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5, required_guarantees=[p_brand in (Brand#23), p_container in (MED BOX)] └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([l_partkey@0], 6), input_partitions=2 + │ RepartitionExec: partitioning=Hash([l_partkey@0], 6), input_partitions=4 │ AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_partkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_partkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── "); Ok(()) @@ -853,44 +781,38 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST] - │ [Stage 4] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + │ SortExec: expr=[o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[c_name@0 as c_name, c_custkey@1 as c_custkey, o_orderkey@2 as o_orderkey, o_orderdate@3 as o_orderdate, o_totalprice@4 as o_totalprice], aggr=[sum(lineitem.l_quantity)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_name@0, c_custkey@1, o_orderkey@2, o_orderdate@3, o_totalprice@4], 6), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[c_name@1 as c_name, c_custkey@0 as c_custkey, o_orderkey@2 as o_orderkey, o_orderdate@4 as o_orderdate, o_totalprice@3 as o_totalprice], aggr=[sum(lineitem.l_quantity)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(l_orderkey@0, o_orderkey@2)] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@2, l_orderkey@0)], projection=[c_custkey@0, c_name@1, o_orderkey@2, o_totalprice@3, o_orderdate@4, l_quantity@6] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_custkey@0, o_custkey@1)], projection=[c_custkey@0, c_name@1, o_orderkey@2, o_totalprice@4, o_orderdate@5] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_name], file_type=parquet + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_custkey, o_totalprice, o_orderdate], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ SortExec: expr=[o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST], preserve_partitioning=[true] - │ AggregateExec: mode=FinalPartitioned, gby=[c_name@0 as c_name, c_custkey@1 as c_custkey, o_orderkey@2 as o_orderkey, o_orderdate@3 as o_orderdate, o_totalprice@4 as o_totalprice], aggr=[sum(lineitem.l_quantity)] - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1 + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: sum(lineitem.l_quantity)@1 > Some(30000),25,2, projection=[l_orderkey@0] + │ AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p17] + ┌───── Stage 1 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([c_name@0, c_custkey@1, o_orderkey@2, o_orderdate@3, o_totalprice@4], 18), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[c_name@1 as c_name, c_custkey@0 as c_custkey, o_orderkey@2 as o_orderkey, o_orderdate@4 as o_orderdate, o_totalprice@3 as o_totalprice], aggr=[sum(lineitem.l_quantity)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(l_orderkey@0, o_orderkey@2)] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@2, l_orderkey@0)], projection=[c_custkey@0, c_name@1, o_orderkey@2, o_totalprice@3, o_orderdate@4, l_quantity@6] - │ CoalescePartitionsExec - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_custkey@0, o_custkey@1)], projection=[c_custkey@0, c_name@1, o_orderkey@2, o_totalprice@4, o_orderdate@5] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_name], file_type=parquet - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_custkey, o_totalprice, o_orderdate], file_type=parquet, predicate=DynamicFilter [ empty ] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] + │ RepartitionExec: partitioning=Hash([l_orderkey@0], 18), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_orderkey, l_quantity], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: sum(lineitem.l_quantity)@1 > Some(30000),25,2, projection=[l_orderkey@0] - │ AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)] - │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([l_orderkey@0], 18), input_partitions=2 - │ AggregateExec: mode=Partial, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_quantity], file_type=parquet - └────────────────────────────────────────────────── "); Ok(()) } @@ -907,16 +829,16 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN ([SM CASE, SM BOX, SM PACK, SM PKG]) AND l_quantity@0 >= Some(100),15,2 AND l_quantity@0 <= Some(1100),15,2 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([MED BAG, MED BOX, MED PKG, MED PACK]) AND l_quantity@0 >= Some(1000),15,2 AND l_quantity@0 <= Some(2000),15,2 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([LG CASE, LG BOX, LG PACK, LG PKG]) AND l_quantity@0 >= Some(2000),15,2 AND l_quantity@0 <= Some(3000),15,2 AND p_size@2 <= 15, projection=[l_extendedprice@6, l_discount@7] │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: (l_quantity@1 >= Some(100),15,2 AND l_quantity@1 <= Some(1100),15,2 OR l_quantity@1 >= Some(1000),15,2 AND l_quantity@1 <= Some(2000),15,2 OR l_quantity@1 >= Some(2000),15,2 AND l_quantity@1 <= Some(3000),15,2) AND (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON, projection=[l_partkey@0, l_quantity@1, l_extendedprice@2, l_discount@3] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], file_type=parquet, predicate=(l_quantity@1 >= Some(100),15,2 AND l_quantity@1 <= Some(1100),15,2 OR l_quantity@1 >= Some(1000),15,2 AND l_quantity@1 <= Some(2000),15,2 OR l_quantity@1 >= Some(2000),15,2 AND l_quantity@1 <= Some(3000),15,2) AND (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON AND DynamicFilter [ empty ], pruning_predicate=(l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(100),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(1100),15,2 OR l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(1000),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(2000),15,2 OR l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(2000),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(3000),15,2) AND (l_shipmode_null_count@6 != row_count@2 AND l_shipmode_min@4 <= AIR AND AIR <= l_shipmode_max@5 OR l_shipmode_null_count@6 != row_count@2 AND l_shipmode_min@4 <= AIR REG AND AIR REG <= l_shipmode_max@5) AND l_shipinstruct_null_count@9 != row_count@2 AND l_shipinstruct_min@7 <= DELIVER IN PERSON AND DELIVER IN PERSON <= l_shipinstruct_max@8, required_guarantees=[l_shipinstruct in (DELIVER IN PERSON), l_shipmode in (AIR, AIR REG)] └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: (p_brand@1 = Brand#12 AND p_container@3 IN ([SM CASE, SM BOX, SM PACK, SM PKG]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([MED BAG, MED BOX, MED PKG, MED PACK]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([LG CASE, LG BOX, LG PACK, LG PKG]) AND p_size@2 <= 15) AND p_size@2 >= 1 - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet, /testdata/tpch/plan_sf0.02/part/10.parquet, /testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet, /testdata/tpch/plan_sf0.02/part/13.parquet, /testdata/tpch/plan_sf0.02/part/14.parquet], [/testdata/tpch/plan_sf0.02/part/15.parquet, /testdata/tpch/plan_sf0.02/part/16.parquet, /testdata/tpch/plan_sf0.02/part/2.parquet], [/testdata/tpch/plan_sf0.02/part/3.parquet, /testdata/tpch/plan_sf0.02/part/4.parquet, /testdata/tpch/plan_sf0.02/part/5.parquet], [/testdata/tpch/plan_sf0.02/part/6.parquet, /testdata/tpch/plan_sf0.02/part/7.parquet, /testdata/tpch/plan_sf0.02/part/8.parquet], ...]}, projection=[p_partkey, p_brand, p_size, p_container], file_type=parquet, predicate=(p_brand@1 = Brand#12 AND p_container@3 IN ([SM CASE, SM BOX, SM PACK, SM PKG]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([MED BAG, MED BOX, MED PKG, MED PACK]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([LG CASE, LG BOX, LG PACK, LG PKG]) AND p_size@2 <= 15) AND p_size@2 >= 1, pruning_predicate=(p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#12 AND Brand#12 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM CASE AND SM CASE <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM BOX AND SM BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM PACK AND SM PACK <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM PKG AND SM PKG <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 5 OR p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BAG AND MED BAG <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED PKG AND MED PKG <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED PACK AND MED PACK <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 10 OR p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#34 AND Brand#34 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG CASE AND LG CASE <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG BOX AND LG BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG PACK AND LG PACK <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG PKG AND LG PKG <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 15) AND p_size_null_count@8 != row_count@3 AND p_size_max@9 >= 1, required_guarantees=[p_brand in (Brand#12, Brand#23, Brand#34), p_container in (LG BOX, LG CASE, LG PACK, LG PKG, MED BAG, MED BOX, MED PACK, MED PKG, SM BOX, SM CASE, SM PACK, SM PKG)] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet], [/testdata/tpch/plan_sf0.02/part/10.parquet], [/testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet], [/testdata/tpch/plan_sf0.02/part/13.parquet], ...]}, projection=[p_partkey, p_brand, p_size, p_container], file_type=parquet, predicate=(p_brand@1 = Brand#12 AND p_container@3 IN ([SM CASE, SM BOX, SM PACK, SM PKG]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([MED BAG, MED BOX, MED PKG, MED PACK]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([LG CASE, LG BOX, LG PACK, LG PKG]) AND p_size@2 <= 15) AND p_size@2 >= 1, pruning_predicate=(p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#12 AND Brand#12 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM CASE AND SM CASE <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM BOX AND SM BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM PACK AND SM PACK <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM PKG AND SM PKG <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 5 OR p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BAG AND MED BAG <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED PKG AND MED PKG <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED PACK AND MED PACK <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 10 OR p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#34 AND Brand#34 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG CASE AND LG CASE <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG BOX AND LG BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG PACK AND LG PACK <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG PKG AND LG PKG <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 15) AND p_size_null_count@8 != row_count@3 AND p_size_max@9 >= 1, required_guarantees=[p_brand in (Brand#12, Brand#23, Brand#34), p_container in (LG BOX, LG CASE, LG PACK, LG PKG, MED BAG, MED BOX, MED PACK, MED PKG, SM BOX, SM CASE, SM PACK, SM PKG)] └────────────────────────────────────────────────── "); Ok(()) @@ -935,7 +857,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[s_suppkey@1, s_name@2, s_address@3] │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ps_partkey@0, l_partkey@1), (ps_suppkey@1, l_suppkey@2)], filter=CAST(ps_availqty@0 AS Float64) > Float64(0.5) * sum(lineitem.l_quantity)@1, projection=[ps_suppkey@1] @@ -943,32 +865,32 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(p_partkey@0, ps_partkey@0)] │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_availqty], file_type=parquet │ ProjectionExec: expr=[0.5 * CAST(sum(lineitem.l_quantity)@2 AS Float64) as Float64(0.5) * sum(lineitem.l_quantity), l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey] │ AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)] │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: n_name@1 = CANADA, projection=[n_nationkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = CANADA, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= CANADA AND CANADA <= n_name_max@1, required_guarantees=[n_name in (CANADA)] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet], [/testdata/tpch/plan_sf0.02/nation/10.parquet], [/testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet], [/testdata/tpch/plan_sf0.02/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = CANADA, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= CANADA AND CANADA <= n_name_max@1, required_guarantees=[n_name in (CANADA)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: p_name@1 LIKE forest%, projection=[p_partkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet, /testdata/tpch/plan_sf0.02/part/10.parquet, /testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet, /testdata/tpch/plan_sf0.02/part/13.parquet, /testdata/tpch/plan_sf0.02/part/14.parquet], [/testdata/tpch/plan_sf0.02/part/15.parquet, /testdata/tpch/plan_sf0.02/part/16.parquet, /testdata/tpch/plan_sf0.02/part/2.parquet], [/testdata/tpch/plan_sf0.02/part/3.parquet, /testdata/tpch/plan_sf0.02/part/4.parquet, /testdata/tpch/plan_sf0.02/part/5.parquet], [/testdata/tpch/plan_sf0.02/part/6.parquet, /testdata/tpch/plan_sf0.02/part/7.parquet, /testdata/tpch/plan_sf0.02/part/8.parquet], ...]}, projection=[p_partkey, p_name], file_type=parquet, predicate=p_name@1 LIKE forest%, pruning_predicate=p_name_null_count@2 != row_count@3 AND p_name_min@0 <= foresu AND forest <= p_name_max@1, required_guarantees=[] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet], [/testdata/tpch/plan_sf0.02/part/10.parquet], [/testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet], [/testdata/tpch/plan_sf0.02/part/13.parquet], ...]}, projection=[p_partkey, p_name], file_type=parquet, predicate=p_name@1 LIKE forest%, pruning_predicate=p_name_null_count@2 != row_count@3 AND p_name_min@0 <= foresu AND forest <= p_name_max@1, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([l_partkey@0, l_suppkey@1], 6), input_partitions=2 + │ RepartitionExec: partitioning=Hash([l_partkey@0, l_suppkey@1], 6), input_partitions=4 │ AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01, projection=[l_partkey@0, l_suppkey@1, l_quantity@2] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01, required_guarantees=[] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01, required_guarantees=[] └────────────────────────────────────────────────── "); Ok(()) @@ -980,56 +902,50 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [numwait@1 DESC, s_name@0 ASC NULLS LAST] - │ [Stage 4] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + │ SortExec: expr=[numwait@1 DESC, s_name@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[s_name@0 as s_name, count(Int64(1))@1 as numwait] + │ AggregateExec: mode=FinalPartitioned, gby=[s_name@0 as s_name], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([s_name@0], 6), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[s_name@0 as s_name], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0, projection=[s_name@0] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0 + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@1)], projection=[s_name@1, l_orderkey@3, l_suppkey@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@0, l_orderkey@2)], projection=[s_name@1, s_nationkey@2, l_orderkey@3, l_suppkey@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_name@1, s_nationkey@2, l_orderkey@3, l_suppkey@4] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@3 > l_commitdate@2 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_suppkey], file_type=parquet + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@3 > l_commitdate@2 └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ SortExec: expr=[numwait@1 DESC, s_name@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[s_name@0 as s_name, count(Int64(1))@1 as numwait] - │ AggregateExec: mode=FinalPartitioned, gby=[s_name@0 as s_name], aggr=[count(Int64(1))] - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1 + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: n_name@1 = SAUDI ARABIA, projection=[n_nationkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet], [/testdata/tpch/plan_sf0.02/nation/10.parquet], [/testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet], [/testdata/tpch/plan_sf0.02/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = SAUDI ARABIA, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= SAUDI ARABIA AND SAUDI ARABIA <= n_name_max@1, required_guarantees=[n_name in (SAUDI ARABIA)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: o_orderstatus@1 = F, projection=[o_orderkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet], [/testdata/tpch/plan_sf0.02/orders/10.parquet], [/testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet], [/testdata/tpch/plan_sf0.02/orders/13.parquet], ...]}, projection=[o_orderkey, o_orderstatus], file_type=parquet, predicate=o_orderstatus@1 = F, pruning_predicate=o_orderstatus_null_count@2 != row_count@3 AND o_orderstatus_min@0 <= F AND F <= o_orderstatus_max@1, required_guarantees=[o_orderstatus in (F)] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([s_name@0], 18), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[s_name@0 as s_name], aggr=[count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0, projection=[s_name@0] - │ CoalescePartitionsExec - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0 - │ CoalescePartitionsExec - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@1)], projection=[s_name@1, l_orderkey@3, l_suppkey@4] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@0, l_orderkey@2)], projection=[s_name@1, s_nationkey@2, l_orderkey@3, l_suppkey@4] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_name@1, s_nationkey@2, l_orderkey@3, l_suppkey@4] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet, /testdata/tpch/plan_sf0.02/supplier/10.parquet, /testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet, /testdata/tpch/plan_sf0.02/supplier/13.parquet, /testdata/tpch/plan_sf0.02/supplier/14.parquet], [/testdata/tpch/plan_sf0.02/supplier/15.parquet, /testdata/tpch/plan_sf0.02/supplier/16.parquet, /testdata/tpch/plan_sf0.02/supplier/2.parquet], [/testdata/tpch/plan_sf0.02/supplier/3.parquet, /testdata/tpch/plan_sf0.02/supplier/4.parquet, /testdata/tpch/plan_sf0.02/supplier/5.parquet], [/testdata/tpch/plan_sf0.02/supplier/6.parquet, /testdata/tpch/plan_sf0.02/supplier/7.parquet, /testdata/tpch/plan_sf0.02/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@3 > l_commitdate@2 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_suppkey], file_type=parquet - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@3 > l_commitdate@2 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: n_name@1 = SAUDI ARABIA, projection=[n_nationkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet, /testdata/tpch/plan_sf0.02/nation/10.parquet, /testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet, /testdata/tpch/plan_sf0.02/nation/13.parquet, /testdata/tpch/plan_sf0.02/nation/14.parquet], [/testdata/tpch/plan_sf0.02/nation/15.parquet, /testdata/tpch/plan_sf0.02/nation/16.parquet, /testdata/tpch/plan_sf0.02/nation/2.parquet], [/testdata/tpch/plan_sf0.02/nation/3.parquet, /testdata/tpch/plan_sf0.02/nation/4.parquet, /testdata/tpch/plan_sf0.02/nation/5.parquet], [/testdata/tpch/plan_sf0.02/nation/6.parquet, /testdata/tpch/plan_sf0.02/nation/7.parquet, /testdata/tpch/plan_sf0.02/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = SAUDI ARABIA, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= SAUDI ARABIA AND SAUDI ARABIA <= n_name_max@1, required_guarantees=[n_name in (SAUDI ARABIA)] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: o_orderstatus@1 = F, projection=[o_orderkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_orderkey, o_orderstatus], file_type=parquet, predicate=o_orderstatus@1 = F, pruning_predicate=o_orderstatus_null_count@2 != row_count@3 AND o_orderstatus_min@0 <= F AND F <= o_orderstatus_max@1, required_guarantees=[o_orderstatus in (F)] - └────────────────────────────────────────────────── "); Ok(()) } @@ -1040,43 +956,37 @@ mod tests { assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [cntrycode@0 ASC NULLS LAST] - │ [Stage 4] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + │ SortExec: expr=[cntrycode@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[cntrycode@0 as cntrycode, count(Int64(1))@1 as numcust, sum(custsale.c_acctbal)@2 as totacctbal] + │ AggregateExec: mode=FinalPartitioned, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cntrycode@0], 6), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)] + │ ProjectionExec: expr=[substr(c_phone@1, 1, 2) as cntrycode, c_acctbal@2 as c_acctbal] + │ NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 > avg(customer.c_acctbal)@0, projection=[avg(customer.c_acctbal)@0, c_phone@1, c_acctbal@2] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(customer.c_acctbal)] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[c_phone@0 as c_phone, c_acctbal@1 as c_acctbal, CAST(c_acctbal@1 AS Decimal128(19, 6)) as join_proj_push_down_1] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(c_custkey@0, o_custkey@0)], projection=[c_phone@1, c_acctbal@2] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_custkey], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] - │ SortExec: expr=[cntrycode@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[cntrycode@0 as cntrycode, count(Int64(1))@1 as numcust, sum(custsale.c_acctbal)@2 as totacctbal] - │ AggregateExec: mode=FinalPartitioned, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)] - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1 + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(customer.c_acctbal)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]), projection=[c_acctbal@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet], [/testdata/tpch/plan_sf0.02/customer/10.parquet], [/testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet], [/testdata/tpch/plan_sf0.02/customer/13.parquet], ...]}, projection=[c_phone, c_acctbal], file_type=parquet, predicate=c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]), pruning_predicate=c_acctbal_null_count@1 != row_count@2 AND c_acctbal_max@0 > Some(0),15,2, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: substr(c_phone@1, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]) + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet], [/testdata/tpch/plan_sf0.02/customer/10.parquet], [/testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet], [/testdata/tpch/plan_sf0.02/customer/13.parquet], ...]}, projection=[c_custkey, c_phone, c_acctbal], file_type=parquet, predicate=substr(c_phone@1, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]) └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p17] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cntrycode@0], 18), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)] - │ ProjectionExec: expr=[substr(c_phone@1, 1, 2) as cntrycode, c_acctbal@2 as c_acctbal] - │ NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 > avg(customer.c_acctbal)@0, projection=[avg(customer.c_acctbal)@0, c_phone@1, c_acctbal@2] - │ AggregateExec: mode=Final, gby=[], aggr=[avg(customer.c_acctbal)] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ ProjectionExec: expr=[c_phone@0 as c_phone, c_acctbal@1 as c_acctbal, CAST(c_acctbal@1 AS Decimal128(19, 6)) as join_proj_push_down_1] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(c_custkey@0, o_custkey@0)], projection=[c_phone@1, c_acctbal@2] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_custkey], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ AggregateExec: mode=Partial, gby=[], aggr=[avg(customer.c_acctbal)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]), projection=[c_acctbal@1] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_phone, c_acctbal], file_type=parquet, predicate=c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]), pruning_predicate=c_acctbal_null_count@1 != row_count@2 AND c_acctbal_max@0 > Some(0),15,2, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] t3:[p6..p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: substr(c_phone@1, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]) - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet, /testdata/tpch/plan_sf0.02/customer/10.parquet, /testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet, /testdata/tpch/plan_sf0.02/customer/13.parquet, /testdata/tpch/plan_sf0.02/customer/14.parquet], [/testdata/tpch/plan_sf0.02/customer/15.parquet, /testdata/tpch/plan_sf0.02/customer/16.parquet, /testdata/tpch/plan_sf0.02/customer/2.parquet], [/testdata/tpch/plan_sf0.02/customer/3.parquet, /testdata/tpch/plan_sf0.02/customer/4.parquet, /testdata/tpch/plan_sf0.02/customer/5.parquet], [/testdata/tpch/plan_sf0.02/customer/6.parquet, /testdata/tpch/plan_sf0.02/customer/7.parquet, /testdata/tpch/plan_sf0.02/customer/8.parquet], ...]}, projection=[c_custkey, c_phone, c_acctbal], file_type=parquet, predicate=substr(c_phone@1, 1, 2) IN ([13, 31, 23, 29, 30, 18, 17]) - └────────────────────────────────────────────────── "); Ok(()) } diff --git a/tests/udfs.rs b/tests/udfs.rs index aea172fc..dd96beaa 100644 --- a/tests/udfs.rs +++ b/tests/udfs.rs @@ -68,12 +68,12 @@ mod tests { assert_snapshot!(physical_distributed_str, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ [Stage 2] => NetworkShuffleExec: output_partitions=1, input_tasks=2 + │ RepartitionExec: partitioning=Hash([test_udf(1)], 1), input_partitions=1 + │ [Stage 1] => NetworkShuffleExec: output_partitions=1, input_tasks=2 └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p0..p1] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([test_udf(1)], 2), input_partitions=1 - │ EmptyExec + │ RepartitionExec: partitioning=Hash([test_udf(1)], 2), input_partitions=1 + │ EmptyExec └────────────────────────────────────────────────── ", ); From d110b813f1b9fb07156cf25a0851d3b7866d0c33 Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Wed, 24 Dec 2025 12:04:07 +0100 Subject: [PATCH 02/27] Bump arrow version (#268) --- Cargo.lock | 69 ++++++++++++++++++++++--------------------- Cargo.toml | 12 ++++---- benchmarks/Cargo.toml | 4 +-- cli/Cargo.toml | 2 +- 4 files changed, 44 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9da3720d..b8412934 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -178,9 +178,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "57.0.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4df8bb5b0bd64c0b9bc61317fcc480bad0f00e56d3bc32c69a4c8dada4786bae" +checksum = "cb372a7cbcac02a35d3fb7b3fc1f969ec078e871f9bb899bf00a2e1809bec8a3" dependencies = [ "arrow-arith", "arrow-array", @@ -199,9 +199,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "57.0.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a640186d3bd30a24cb42264c2dafb30e236a6f50d510e56d40b708c9582491" +checksum = "0f377dcd19e440174596d83deb49cd724886d91060c07fec4f67014ef9d54049" dependencies = [ "arrow-array", "arrow-buffer", @@ -213,9 +213,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "57.0.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219fe420e6800979744c8393b687afb0252b3f8a89b91027d27887b72aa36d31" +checksum = "a23eaff85a44e9fa914660fb0d0bb00b79c4a3d888b5334adb3ea4330c84f002" dependencies = [ "ahash", "arrow-buffer", @@ -232,9 +232,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "57.0.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76885a2697a7edf6b59577f568b456afc94ce0e2edc15b784ce3685b6c3c5c27" +checksum = "a2819d893750cb3380ab31ebdc8c68874dd4429f90fd09180f3c93538bd21626" dependencies = [ "bytes", "half", @@ -244,13 +244,14 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "57.0.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9ebb4c987e6b3b236fb4a14b20b34835abfdd80acead3ccf1f9bf399e1f168" +checksum = "e3d131abb183f80c450d4591dc784f8d7750c50c6e2bc3fcaad148afc8361271" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", + "arrow-ord", "arrow-schema", "arrow-select", "atoi", @@ -265,9 +266,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "57.0.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92386159c8d4bce96f8bd396b0642a0d544d471bdc2ef34d631aec80db40a09c" +checksum = "2275877a0e5e7e7c76954669366c2aa1a829e340ab1f612e647507860906fb6b" dependencies = [ "arrow-array", "arrow-cast", @@ -280,9 +281,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "57.0.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727681b95de313b600eddc2a37e736dcb21980a40f640314dcf360e2f36bc89b" +checksum = "05738f3d42cb922b9096f7786f606fcb8669260c2640df8490533bb2fa38c9d3" dependencies = [ "arrow-buffer", "arrow-schema", @@ -293,9 +294,9 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "57.0.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f70bb56412a007b0cfc116d15f24dda6adeed9611a213852a004cda20085a3b9" +checksum = "8b5f57c3d39d1b1b7c1376a772ea86a131e7da310aed54ebea9363124bb885e3" dependencies = [ "arrow-array", "arrow-buffer", @@ -313,9 +314,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "57.0.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da9ba92e3de170295c98a84e5af22e2b037f0c7b32449445e6c493b5fca27f27" +checksum = "3d09446e8076c4b3f235603d9ea7c5494e73d441b01cd61fb33d7254c11964b3" dependencies = [ "arrow-array", "arrow-buffer", @@ -329,9 +330,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "57.0.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b969b4a421ae83828591c6bf5450bd52e6d489584142845ad6a861f42fe35df8" +checksum = "371ffd66fa77f71d7628c63f209c9ca5341081051aa32f9c8020feb0def787c0" dependencies = [ "arrow-array", "arrow-buffer", @@ -353,9 +354,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "57.0.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "141c05298b21d03e88062317a1f1a73f5ba7b6eb041b350015b1cd6aabc0519b" +checksum = "cbc94fc7adec5d1ba9e8cd1b1e8d6f72423b33fe978bf1f46d970fafab787521" dependencies = [ "arrow-array", "arrow-buffer", @@ -366,9 +367,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "57.0.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f3c06a6abad6164508ed283c7a02151515cef3de4b4ff2cebbcaeb85533db2" +checksum = "169676f317157dc079cc5def6354d16db63d8861d61046d2f3883268ced6f99f" dependencies = [ "arrow-array", "arrow-buffer", @@ -379,9 +380,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "57.0.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cfa7a03d1eee2a4d061476e1840ad5c9867a544ca6c4c59256496af5d0a8be5" +checksum = "d27609cd7dd45f006abae27995c2729ef6f4b9361cde1ddd019dc31a5aa017e0" dependencies = [ "serde_core", "serde_json", @@ -389,9 +390,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "57.0.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bafa595babaad59f2455f4957d0f26448fb472722c186739f4fac0823a1bdb47" +checksum = "ae980d021879ea119dd6e2a13912d81e64abed372d53163e804dfe84639d8010" dependencies = [ "ahash", "arrow-array", @@ -403,9 +404,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "57.0.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f46457dbbb99f2650ff3ac23e46a929e0ab81db809b02aa5511c258348bef2" +checksum = "cf35e8ef49dcf0c5f6d175edee6b8af7b45611805333129c541a8b89a0fc0534" dependencies = [ "arrow-array", "arrow-buffer", @@ -3336,9 +3337,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4_flex" -version = "0.11.5" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" +checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e" dependencies = [ "twox-hash", ] @@ -3594,9 +3595,9 @@ dependencies = [ [[package]] name = "parquet" -version = "57.0.0" +version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a0f31027ef1af7549f7cec603a9a21dce706d3f8d7c2060a68f43c1773be95a" +checksum = "be3e4f6d320dd92bfa7d612e265d7d08bba0a240bab86af3425e1d255a511d89" dependencies = [ "ahash", "arrow-array", diff --git a/Cargo.toml b/Cargo.toml index 3868f544..8f8490ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,8 +19,8 @@ datafusion = { workspace = true, features = [ "datetime_expressions", ] } datafusion-proto = { workspace = true } -arrow-flight = "57.0.0" -arrow-select = "57.0.0" +arrow-flight = "57.1.0" +arrow-select = "57.1.0" async-trait = "0.1.88" tokio = { version = "1.46.1", features = ["full"] } tonic = { version = "0.14.1", features = ["transport"] } @@ -43,8 +43,8 @@ tokio-stream = "0.1.17" insta = { version = "1.43.1", features = ["filters"], optional = true } tpchgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "e83365a5a9101906eb9f78c5607b83bc59849acf", optional = true } tpchgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "e83365a5a9101906eb9f78c5607b83bc59849acf", optional = true } -parquet = { version = "57.0.0", optional = true } -arrow = { version = "57.0.0", optional = true } +parquet = { version = "57.1.0", optional = true } +arrow = { version = "57.1.0", optional = true } hyper-util = { version = "0.1.16", optional = true } pretty_assertions = { version = "1.4", optional = true } @@ -68,8 +68,8 @@ structopt = "0.3" insta = { version = "1.43.1", features = ["filters"] } tpchgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "e83365a5a9101906eb9f78c5607b83bc59849acf" } tpchgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "e83365a5a9101906eb9f78c5607b83bc59849acf" } -parquet = "57.0.0" -arrow = "57.0.0" +parquet = "57.1.0" +arrow = "57.1.0" tokio-stream = "0.1.17" hyper-util = "0.1.16" pretty_assertions = "1.4" diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 1a30ae2b..77f481a9 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -9,7 +9,7 @@ datafusion = { workspace = true } datafusion-proto = { workspace = true } datafusion-distributed = { path = "..", features = ["integration"] } tokio = { version = "1.46.1", features = ["full"] } -parquet = { version = "57.0.0" } +parquet = { version = "57.1.0" } structopt = { version = "0.3.26" } log = "0.4.27" serde = "1.0.219" @@ -21,7 +21,7 @@ futures = "0.3.31" dashmap = "6.1.0" prost = "0.14.0" url = "2.5.4" -arrow-flight = "57.0.0" +arrow-flight = "57.1.0" tonic = { version = "0.14.1", features = ["transport"] } axum = "0.7" object_store = { version = "0.12.4", features = ["aws"] } diff --git a/cli/Cargo.toml b/cli/Cargo.toml index ee76ce2e..bd23b27d 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -11,7 +11,7 @@ tokio = { version = "1.46.1", features = ["full"] } clap = { version = "4", features = ["derive"] } env_logger = "0.11" dirs = "6" -arrow-flight = "57.0.0" +arrow-flight = "57.1.0" tonic = { version = "0.14.1", features = ["transport"] } tower = "0.5.2" hyper-util = "0.1.16" From b7e5555b0dbbea3c77d1dc47f5d07cdda87517df Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Fri, 26 Dec 2025 08:50:40 +0100 Subject: [PATCH 03/27] Make queries in TPC-DS get distributed (#264) * Make TPC-DS tests use DataFusion test dataset * Delete unnecessary scripts and tweak pipelines * Add TPC-DS plan assertion tests * Answer to feedback * Do not use multithreading --- .github/workflows/ci.yml | 16 +- .gitignore | 4 +- Cargo.lock | 340 +- Cargo.toml | 6 + src/test_utils/property_based.rs | 16 +- src/test_utils/tpcds.rs | 430 +- testdata/tpcds/README.md | 2 - testdata/tpcds/generate.sh | 70 - ...pcds_test.rs => tpcds_correctness_test.rs} | 149 +- tests/tpcds_plans_test.rs | 10243 ++++++++++++++++ 10 files changed, 10868 insertions(+), 408 deletions(-) delete mode 100755 testdata/tpcds/generate.sh rename tests/{tpcds_test.rs => tpcds_correctness_test.rs} (75%) create mode 100644 tests/tpcds_plans_test.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad7d057f..ca322307 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,18 +52,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - with: - lfs: true - uses: ./.github/actions/setup - - name: Install DuckDB CLI - run: | - curl https://install.duckdb.org | sh - mkdir -p $HOME/.local/bin - mv /home/runner/.duckdb/cli/latest/duckdb $HOME/.local/bin/ - echo "$HOME/.local/bin" >> $GITHUB_PATH - - name: Run TPC-DS test - id: test - run: cargo test --features tpcds --test tpcds_test + - uses: actions/cache@v4 + with: + path: testdata/tpcds/main.zip + key: "main.zip" + - run: cargo test --features tpcds --test 'tpcds_*' format-check: runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index f1c1dc7c..07512695 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,6 @@ /benchmarks/data/ testdata/tpch/* !testdata/tpch/queries -testdata/tpcds/data/ +testdata/tpcds/* +!testdata/tpcds/queries +!testdata/tpcds/README.md diff --git a/Cargo.lock b/Cargo.lock index b8412934..0d61ef9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + [[package]] name = "ahash" version = "0.8.12" @@ -164,6 +175,15 @@ dependencies = [ "object", ] +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arrayref" version = "0.3.9" @@ -1210,6 +1230,16 @@ dependencies = [ "phf", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "2.34.0" @@ -1337,6 +1367,16 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -1362,6 +1402,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.5.0" @@ -1813,6 +1868,7 @@ dependencies = [ "pretty_assertions", "prost", "rand 0.8.5", + "reqwest", "structopt", "tokio", "tokio-stream", @@ -1822,6 +1878,7 @@ dependencies = [ "tpchgen-arrow", "url", "uuid", + "zip", ] [[package]] @@ -2285,6 +2342,12 @@ dependencies = [ "sqlparser", ] +[[package]] +name = "deflate64" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26bf8fc351c5ed29b5c2f0cbbac1b209b74f60ecd62e675a998df72c49af5204" + [[package]] name = "delegate" version = "0.13.5" @@ -2305,6 +2368,17 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.110", +] + [[package]] name = "diff" version = "0.1.13" @@ -2372,6 +2446,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "endian-type" version = "0.1.2" @@ -2485,6 +2568,21 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -2927,6 +3025,22 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.8.1", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.18" @@ -2946,9 +3060,11 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2 0.6.1", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -3093,6 +3209,15 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "insta" version = "1.44.1" @@ -3344,6 +3469,16 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "lzma-rust2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c60a23ffb90d527e23192f1246b14746e2f7f071cb84476dd879071696c18a4a" +dependencies = [ + "crc", + "sha2", +] + [[package]] name = "lzma-sys" version = "0.1.20" @@ -3419,6 +3554,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework 2.11.1", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nibble_vec" version = "0.1.0" @@ -3543,12 +3695,50 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "openssl" +version = "0.10.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +dependencies = [ + "bitflags 2.10.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.110", +] + [[package]] name = "openssl-probe" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "openssl-sys" +version = "0.9.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -3637,6 +3827,16 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -3741,6 +3941,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppmd-rust" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d558c559f0450f16f2a27a1f017ef38468c1090c9ce63c8e51366232d53717b4" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -4098,6 +4304,7 @@ checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" dependencies = [ "base64", "bytes", + "encoding_rs", "futures-core", "futures-util", "h2 0.4.12", @@ -4106,9 +4313,12 @@ dependencies = [ "http-body-util", "hyper 1.8.1", "hyper-rustls 0.27.7", + "hyper-tls", "hyper-util", "js-sys", "log", + "mime", + "native-tls", "percent-encoding", "pin-project-lite", "quinn", @@ -4120,6 +4330,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", + "tokio-native-tls", "tokio-rustls 0.26.4", "tokio-util", "tower", @@ -4239,7 +4450,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 3.5.1", ] [[package]] @@ -4351,6 +4562,19 @@ dependencies = [ "untrusted", ] +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + [[package]] name = "security-framework" version = "3.5.1" @@ -4358,7 +4582,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" dependencies = [ "bitflags 2.10.0", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -4462,6 +4686,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.9" @@ -4693,6 +4928,27 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempfile" version = "3.23.0" @@ -4838,6 +5094,16 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.24.1" @@ -5137,6 +5403,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "vec_map" version = "0.8.2" @@ -5352,6 +5624,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -5646,6 +5929,20 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.110", +] [[package]] name = "zerotrie" @@ -5680,12 +5977,51 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "zip" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" +dependencies = [ + "aes", + "arbitrary", + "bzip2 0.6.1", + "constant_time_eq", + "crc32fast", + "deflate64", + "flate2", + "getrandom 0.3.4", + "hmac", + "indexmap", + "lzma-rust2", + "memchr", + "pbkdf2", + "ppmd-rust", + "sha1", + "time", + "zeroize", + "zopfli", + "zstd", +] + [[package]] name = "zlib-rs" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2" +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + [[package]] name = "zstd" version = "0.13.3" diff --git a/Cargo.toml b/Cargo.toml index 8f8490ff..71cce7fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,6 +47,8 @@ parquet = { version = "57.1.0", optional = true } arrow = { version = "57.1.0", optional = true } hyper-util = { version = "0.1.16", optional = true } pretty_assertions = { version = "1.4", optional = true } +reqwest = { version = "0.12", optional = true } +zip = { version = "6.0", optional = true } [features] avro = ["datafusion/avro"] @@ -58,6 +60,8 @@ integration = [ "arrow", "hyper-util", "pretty_assertions", + "reqwest", + "zip" ] tpch = ["integration"] @@ -73,3 +77,5 @@ arrow = "57.1.0" tokio-stream = "0.1.17" hyper-util = "0.1.16" pretty_assertions = "1.4" +reqwest = "0.12" +zip = "6.0" diff --git a/src/test_utils/property_based.rs b/src/test_utils/property_based.rs index 3aabe112..8c7e2974 100644 --- a/src/test_utils/property_based.rs +++ b/src/test_utils/property_based.rs @@ -12,7 +12,7 @@ use datafusion::{ use std::sync::Arc; /// compares the set of record batches for equality -pub async fn compare_result_set( +pub fn compare_result_set( actual_result: &Result>, expected_result: &Result>, ) -> Result<()> { @@ -42,7 +42,7 @@ pub async fn compare_result_set( // Ensures that the plans have the same ordering properties and that the actual result is sorted // correctly. -pub async fn compare_ordering( +pub fn compare_ordering( actual_physical_plan: Arc, expected_physical_plan: Arc, actual_result: &Result>, @@ -448,18 +448,10 @@ mod tests { let df = expected_ctx.sql(ordered_query).await.unwrap(); let expected_plan = df.create_physical_plan().await.unwrap(); - assert!( - compare_ordering(actual_plan.clone(), expected_plan.clone(), &results) - .await - .is_ok() - ); + assert!(compare_ordering(actual_plan.clone(), expected_plan.clone(), &results).is_ok()); // This should fail because the batch is not sorted by value let result = Ok(vec![batch]); - assert!( - compare_ordering(actual_plan.clone(), expected_plan.clone(), &result) - .await - .is_err() - ); + assert!(compare_ordering(actual_plan.clone(), expected_plan.clone(), &result).is_err()); } } diff --git a/src/test_utils/tpcds.rs b/src/test_utils/tpcds.rs index 4aa3288e..017eb9bf 100644 --- a/src/test_utils/tpcds.rs +++ b/src/test_utils/tpcds.rs @@ -1,35 +1,24 @@ -use datafusion::{ - arrow::{ - array::{Array, ArrayRef, DictionaryArray, StringArray, StringViewArray}, - datatypes::{DataType, Field, Schema, UInt16Type}, - record_batch::RecordBatch, - }, - common::{internal_datafusion_err, internal_err}, - error::Result, - execution::context::SessionContext, - prelude::ParquetReadOptions, -}; -use parquet::{arrow::ArrowWriter, file::properties::WriterProperties}; +use arrow::datatypes::{DataType, Field}; +use datafusion::common::{internal_datafusion_err, internal_err}; +use datafusion::error::DataFusionError; +use datafusion::physical_expr::Partitioning; +use datafusion::physical_expr::expressions::{CastColumnExpr, Column}; +use datafusion::physical_expr::projection::ProjectionExpr; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::projection::ProjectionExec; +use datafusion::physical_plan::repartition::RepartitionExec; +use datafusion::prelude::{ParquetReadOptions, SessionContext}; +use parquet::file::properties::WriterProperties; use std::fs; -use std::path::Path; -use std::process::Command; +use std::io::Write; +use std::path::{Path, PathBuf}; use std::sync::Arc; -pub fn get_data_dir() -> std::path::PathBuf { - std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/tpcds/data") -} - -pub fn get_queries_dir() -> std::path::PathBuf { - std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/tpcds/queries") -} - -pub fn get_tpcds_dir() -> std::path::PathBuf { - std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/tpcds") -} +const URL: &str = "https://github.com/apache/datafusion-benchmarks/archive/refs/heads/main.zip"; /// Load a single TPC-DS query by ID (1-99). -pub fn get_test_tpcds_query(id: usize) -> Result { - let queries_dir = get_queries_dir(); +pub fn get_test_tpcds_query(id: usize) -> Result { + let queries_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/tpcds/queries"); if !queries_dir.exists() { return internal_err!( @@ -46,7 +35,7 @@ pub fn get_test_tpcds_query(id: usize) -> Result { let query_sql = fs::read_to_string(&query_file) .map_err(|e| { - internal_datafusion_err!("Failed to read query file {}: {}", query_file.display(), e) + internal_datafusion_err!("Failed to read query file {}: {e}", query_file.display()) })? .trim() .to_string(); @@ -54,251 +43,210 @@ pub fn get_test_tpcds_query(id: usize) -> Result { Ok(query_sql) } -pub const TPCDS_TABLES: &[&str] = &[ - "call_center", - "catalog_page", - "catalog_returns", - "catalog_sales", - "customer", - "customer_address", - "customer_demographics", - "date_dim", - "household_demographics", - "income_band", - "inventory", - "item", - "promotion", - "reason", - "ship_mode", - "store", - "store_returns", - "store_sales", - "time_dim", - "warehouse", - "web_page", - "web_returns", - "web_sales", - "web_site", -]; +/// Downloads the datafusion-benchmarks repository as a zip file +async fn download_benchmarks(dest_path: PathBuf) -> Result<(), Box> { + if dest_path.exists() { + return Ok(()); + } -/// Tables that should have dictionary encoding applied for testing -const DICT_ENCODING_TABLES: &[&str] = &["item", "customer", "store"]; + // Create directory if it doesn't exist + if let Some(parent) = dest_path.parent() { + fs::create_dir_all(parent)?; + } -/// Force dictionary encoding for specific string columns in a table for extra test coverage. -fn force_dictionary_encoding_for_table( - table_name: &str, - batch: RecordBatch, -) -> Result { - let dict_columns = match table_name { - "item" => vec!["i_brand", "i_category", "i_class", "i_color", "i_size"], - "customer" => vec!["c_salutation"], - "store" => vec!["s_state", "s_country"], - _ => vec![], // No dictionary encoding for other tables - }; + // Download the file + let response = reqwest::get(URL).await?; + let bytes = response.bytes().await?; - if dict_columns.is_empty() { - return Ok(batch); - } + // Write to file + let mut file = fs::File::create(&dest_path)?; + file.write_all(&bytes)?; - let schema = batch.schema(); - let mut new_fields = Vec::new(); - let mut new_columns = Vec::new(); + Ok(()) +} - for (i, field) in schema.fields().iter().enumerate() { - let column = batch.column(i); +/// Unzips the downloaded benchmarks zip file +fn unzip_benchmarks( + zip_path: PathBuf, + extract_to: PathBuf, +) -> Result<(), Box> { + if extract_to.exists() { + return Ok(()); + } - // Check if this column should be dictionary-encoded - if dict_columns.contains(&field.name().as_str()) - && matches!(field.data_type(), DataType::Utf8 | DataType::Utf8View) - { - // Convert to dictionary encoding - let string_data = - if let Some(string_array) = column.as_any().downcast_ref::() { - string_array.iter().collect::>() - } else if let Some(view_array) = column.as_any().downcast_ref::() { - view_array.iter().collect::>() - } else { - return internal_err!("Expected string array for column {}", field.name()); - }; + let file = fs::File::open(zip_path)?; + let mut archive = zip::ZipArchive::new(file)?; - let dict_array: DictionaryArray = string_data.into_iter().collect(); - let dict_field = Field::new( - field.name(), - DataType::Dictionary(Box::new(DataType::UInt16), Box::new(DataType::Utf8)), - field.is_nullable(), - ); + for i in 0..archive.len() { + let mut zip_file = archive.by_index(i)?; + let file_name = zip_file.name(); + if !(file_name.contains("tpcds") && file_name.ends_with(".parquet")) { + continue; + } + let outpath = extract_to.join(zip_file.mangled_name().file_name().unwrap()); - new_fields.push(dict_field); - new_columns.push(Arc::new(dict_array) as ArrayRef); - } else { - new_fields.push((**field).clone()); - new_columns.push(column.clone()); + if let Some(parent) = outpath.parent() { + fs::create_dir_all(parent)?; } + let mut outfile = fs::File::create(&outpath)?; + std::io::copy(&mut zip_file, &mut outfile)?; } - let new_schema = Arc::new(Schema::new(new_fields)); - RecordBatch::try_new(new_schema, new_columns).map_err(|e| internal_datafusion_err!("{}", e)) -} - -pub async fn register_tpcds_table( - ctx: &SessionContext, - table_name: &str, - data_dir: Option<&Path>, -) -> Result<()> { - register_tpcds_table_with_options(ctx, table_name, data_dir, false).await + Ok(()) } -pub async fn register_tpcds_table_with_options( - ctx: &SessionContext, - table_name: &str, - data_dir: Option<&Path>, - dict_encode_items_table: bool, -) -> Result<()> { - let default_data_dir = get_data_dir(); - let data_path = data_dir.unwrap_or(&default_data_dir); - - // Apply dictionary encoding if requested and materialize to disk - if dict_encode_items_table && DICT_ENCODING_TABLES.contains(&table_name) { - let table_dir_path = data_path.join(table_name); - if table_dir_path.is_dir() { - let dict_table_path = data_path.join(format!("{table_name}_dict")); - - // Check if dictionary encoded version already exists - if dict_table_path.exists() { - // Use the existing dictionary encoded version - ctx.register_parquet( - table_name, - &dict_table_path.to_string_lossy(), - ParquetReadOptions::default(), - ) - .await?; - return Ok(()); - } - - // Register temporarily to read the original data - let temp_table_name = format!("temp_{table_name}"); - ctx.register_parquet( - &temp_table_name, - &table_dir_path.to_string_lossy(), - ParquetReadOptions::default(), - ) - .await?; - - // Read data and apply dictionary encoding - let df = ctx.table(&temp_table_name).await?; - let batches = df.collect().await?; - - let mut dict_batches = Vec::new(); - for batch in batches { - dict_batches.push(force_dictionary_encoding_for_table(table_name, batch)?); - } - - // Write dictionary-encoded data to disk - if !dict_batches.is_empty() { - fs::create_dir_all(&dict_table_path)?; - let dict_file_path = dict_table_path.join("data.parquet"); - let file = fs::File::create(&dict_file_path)?; - let props = WriterProperties::builder().build(); - let mut writer = ArrowWriter::try_new(file, dict_batches[0].schema(), Some(props))?; - - for batch in &dict_batches { - writer.write(batch)?; - } - writer.close()?; - - // Register the dictionary encoded table - ctx.register_parquet( - table_name, - &dict_table_path.to_string_lossy(), - ParquetReadOptions::default(), - ) - .await?; - } +async fn repartition_parquet_file( + file_path: PathBuf, + dest_path: PathBuf, + partitions: usize, + use_dict_encoding: bool, +) -> Result<(), DataFusionError> { + if !file_path.exists() { + return internal_err!("Path {} does not exist", file_path.display()); + } + let file_name = file_path.file_name().unwrap().to_str().unwrap(); + if !file_name.ends_with(".parquet") { + return internal_err!("Path {} is not parquet", file_path.display()); + } + let table_name = file_name.trim_end_matches(".parquet"); - // Deregister the temporary table - ctx.deregister_table(&temp_table_name)?; + if let Ok(dir) = fs::read_dir(&dest_path) { + if dir.count() >= 1 { return Ok(()); } } - // Use normal parquet registration for all tables - let table_file_path = data_path.join(format!("{table_name}.parquet")); - if table_file_path.is_file() { - ctx.register_parquet( - table_name, - &table_file_path.to_string_lossy(), - ParquetReadOptions::default(), - ) + let ctx = SessionContext::new(); + ctx.sql("SET datafusion.execution.target_partitions=1") .await?; - return Ok(()); - } - // Check if this is a directory with multiple parquet files - let table_dir_path = data_path.join(table_name); - if table_dir_path.is_dir() { - ctx.register_parquet( - table_name, - &table_dir_path.to_string_lossy(), - ParquetReadOptions::default(), - ) - .await?; - return Ok(()); + ctx.register_parquet( + table_name, + &file_path.to_str().unwrap(), + ParquetReadOptions::default(), + ) + .await?; + + let table = ctx.table(table_name).await?; + let mut plan = table.create_physical_plan().await?; + if use_dict_encoding && table_name == "item" { + let cols = ["i_brand", "i_category", "i_class", "i_color", "i_size"]; + plan = project_cols_as_dict(plan, &cols)?; + } else if use_dict_encoding && table_name == "customer" { + let cols = ["c_salutation"]; + plan = project_cols_as_dict(plan, &cols)?; + } else if use_dict_encoding && table_name == "store" { + let cols = ["s_state", "s_country"]; + plan = project_cols_as_dict(plan, &cols)?; } - internal_err!( - "TPCDS table not found: {} (looked for both file and directory)", - table_name + let plan = RepartitionExec::try_new(plan, Partitioning::RoundRobinBatch(partitions))?; + ctx.write_parquet( + Arc::new(plan), + dest_path.to_str().unwrap(), + Some( + WriterProperties::builder() + .set_dictionary_enabled(true) + .build(), + ), ) -} + .await?; -pub async fn register_tables(ctx: &SessionContext) -> Result> { - register_tables_with_options(ctx, false).await + Ok(()) } -pub async fn register_tables_with_options( - ctx: &SessionContext, - dict_encode_items_table: bool, -) -> Result> { - let mut registered_tables = Vec::new(); +fn project_cols_as_dict( + plan: Arc, + cols: &[&str], +) -> Result, DataFusionError> { + let project = ProjectionExec::try_new( + plan.schema() + .fields + .iter() + .enumerate() + .map(|(i, f)| ProjectionExpr { + expr: if cols.contains(&f.name().as_str()) { + Arc::new(CastColumnExpr::new( + Arc::new(Column::new(f.name(), i)), + f.clone(), + Arc::new(Field::new( + f.name(), + DataType::Dictionary( + Box::new(DataType::UInt16), + Box::new(DataType::Utf8), + ), + f.is_nullable(), + )), + None, + )) + } else { + Arc::new(Column::new(f.name(), i)) + }, + alias: f.name().to_string(), + }), + plan, + )?; + Ok(Arc::new(project)) +} - for &table_name in TPCDS_TABLES { - register_tpcds_table_with_options(ctx, table_name, None, dict_encode_items_table).await?; - registered_tables.push(table_name.to_string()); +pub async fn prepare_tables( + data_path: PathBuf, + dest_path: PathBuf, + partitions: usize, +) -> datafusion::common::Result<()> { + for entry in fs::read_dir(data_path)? { + let entry = entry?; + let file_name = entry.file_name(); + let file_name = file_name.to_str().unwrap(); + if !file_name.ends_with(".parquet") { + continue; + } + let table_name = file_name.trim_end_matches(".parquet"); + // Apply dictionary encoding if requested and materialize to disk + /// Tables that should have dictionary encoding applied for testing + const DICT_ENCODING_TABLES: &[&str] = &["item", "customer", "store"]; + + repartition_parquet_file( + entry.path(), + dest_path.join(table_name), + partitions, + DICT_ENCODING_TABLES.contains(&table_name), + ) + .await?; } - - Ok(registered_tables) + Ok(()) } -/// Generate TPC-DS data using the generation script -pub fn generate_tpcds_data(scale_factor: &str) -> Result<()> { - let tpcds_dir = get_tpcds_dir(); - let generate_script = tpcds_dir.join("generate.sh"); - - if !generate_script.exists() { - return internal_err!( - "TPC-DS generation script not found: {}", - generate_script.display() - ); +pub async fn generate_tpcds_data( + dir: &Path, + sf: f64, + partitions: usize, +) -> Result<(), Box> { + if sf != 1.0 { + Err("Only scale factor 1.0 is supported for TPC-DS")?; } + let base_path = dir.parent().unwrap(); + download_benchmarks(base_path.join("main.zip")).await?; + unzip_benchmarks(base_path.join("main.zip"), base_path.join("downloaded"))?; + prepare_tables(base_path.join("downloaded"), dir.to_path_buf(), partitions).await?; + Ok(()) +} - let output = Command::new("bash") - .arg(&generate_script) - .arg(scale_factor) - .current_dir(&tpcds_dir) - .output() - .map_err(|e| { - internal_datafusion_err!("Failed to execute TPC-DS generation script: {}", e) - })?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - let stdout = String::from_utf8_lossy(&output.stdout); - return internal_err!( - "TPC-DS generation failed:\nstdout: {}\nstderr: {}", - stdout, - stderr - ); +pub async fn register_tables( + ctx: &SessionContext, + data_path: &Path, +) -> Result<(), DataFusionError> { + for entry in fs::read_dir(data_path)? { + let path = entry?.path(); + if path.is_dir() { + let table_name = path.file_name().unwrap().to_str().unwrap(); + ctx.register_parquet( + table_name, + path.to_str().unwrap(), + ParquetReadOptions::default(), + ) + .await?; + } } - Ok(()) } diff --git a/testdata/tpcds/README.md b/testdata/tpcds/README.md index fad29fe6..56c5a0eb 100644 --- a/testdata/tpcds/README.md +++ b/testdata/tpcds/README.md @@ -4,5 +4,3 @@ This directory contains 99 TPC-DS queries from https://github.com/duckdb/duckdb - Queries 47 and 57 were modified to add explicit ORDER BY d_moy to avg() window function. DataFusion requires explicit ordering in window functions with PARTITION BY for deterministic results. - Query 72 was modified to support date functions in datafusion - -`generate.sh {SCALE_FACTOR}` is a script which can generate TPC-DS parquet data. Requires the duckdb CLI: https://duckdb.org/install/ diff --git a/testdata/tpcds/generate.sh b/testdata/tpcds/generate.sh deleted file mode 100755 index 4beeb0ac..00000000 --- a/testdata/tpcds/generate.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash - -set -e - -# Get the directory where this script is located -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -if [ $# -ne 1 ]; then - echo "Usage: $0 " - echo "Scale factor must be greater than or equal to 0" - exit 1 -fi - -SCALE_FACTOR=$1 - -if ! [[ "$SCALE_FACTOR" =~ ^[0-9]+(\.[0-9]+)?$ ]] || (( $(echo "$SCALE_FACTOR < 0" | bc -l) )); then - echo "Error: Scale factor must be a number greater than or equal to 0" - exit 1 -fi - -if ! command -v duckdb &> /dev/null; then - echo "Error: duckdb CLI is not installed" - echo "Please install duckdb: https://duckdb.org/docs/installation/" - exit 1 -fi - -echo "Clearing testdata/tpcds/data directory..." -rm -rf "$SCRIPT_DIR/data" -mkdir "$SCRIPT_DIR/data" - -echo "Removing existing database file..." -rm -f "$SCRIPT_DIR/tpcds.duckdb" - -echo "Generating TPC-DS data with scale factor $SCALE_FACTOR..." -duckdb "$SCRIPT_DIR/tpcds.duckdb" -c "INSTALL tpcds; LOAD tpcds; CALL dsdgen(sf=$SCALE_FACTOR);" - -FILE_SIZE_BYTES=16777216 - -echo "Exporting tables to parquet files..." -duckdb "$SCRIPT_DIR/tpcds.duckdb" << EOF -COPY store_sales TO '$SCRIPT_DIR/data/store_sales' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY catalog_sales TO '$SCRIPT_DIR/data/catalog_sales' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY web_sales TO '$SCRIPT_DIR/data/web_sales' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY store_returns TO '$SCRIPT_DIR/data/store_returns' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY catalog_returns TO '$SCRIPT_DIR/data/catalog_returns' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY web_returns TO '$SCRIPT_DIR/data/web_returns' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY inventory TO '$SCRIPT_DIR/data/inventory' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY date_dim TO '$SCRIPT_DIR/data/date_dim' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY time_dim TO '$SCRIPT_DIR/data/time_dim' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY item TO '$SCRIPT_DIR/data/item' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY customer TO '$SCRIPT_DIR/data/customer' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY customer_address TO '$SCRIPT_DIR/data/customer_address' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY customer_demographics TO '$SCRIPT_DIR/data/customer_demographics' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY household_demographics TO '$SCRIPT_DIR/data/household_demographics' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY income_band TO '$SCRIPT_DIR/data/income_band' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY warehouse TO '$SCRIPT_DIR/data/warehouse' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY store TO '$SCRIPT_DIR/data/store' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY call_center TO '$SCRIPT_DIR/data/call_center' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY web_site TO '$SCRIPT_DIR/data/web_site' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY web_page TO '$SCRIPT_DIR/data/web_page' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY ship_mode TO '$SCRIPT_DIR/data/ship_mode' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY reason TO '$SCRIPT_DIR/data/reason' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY promotion TO '$SCRIPT_DIR/data/promotion' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -COPY catalog_page TO '$SCRIPT_DIR/data/catalog_page' (FORMAT PARQUET, COMPRESSION ZSTD, FILE_SIZE_BYTES ${FILE_SIZE_BYTES}); -EOF - -echo "Cleaning up temporary database..." -rm -f "$SCRIPT_DIR/tpcds.duckdb" - -echo "TPC-DS data generation complete!" diff --git a/tests/tpcds_test.rs b/tests/tpcds_correctness_test.rs similarity index 75% rename from tests/tpcds_test.rs rename to tests/tpcds_correctness_test.rs index 5f9a1aa2..879486a3 100644 --- a/tests/tpcds_test.rs +++ b/tests/tpcds_correctness_test.rs @@ -1,44 +1,28 @@ #[cfg(all(feature = "integration", feature = "tpcds", test))] mod tests { - use datafusion::common::runtime::JoinSet; + use datafusion::arrow::array::RecordBatch; + use datafusion::common::plan_err; use datafusion::error::Result; use datafusion::physical_plan::{ExecutionPlan, collect}; use datafusion::prelude::SessionContext; - use datafusion_distributed::test_utils::{ - localhost::start_localhost_context, - property_based::{compare_ordering, compare_result_set}, - tpcds::{ - generate_tpcds_data, get_data_dir, get_test_tpcds_query, register_tables_with_options, - }, + use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::test_utils::property_based::{ + compare_ordering, compare_result_set, + }; + use datafusion_distributed::test_utils::tpcds; + use datafusion_distributed::{ + DefaultSessionBuilder, DistributedExec, DistributedExt, display_plan_ascii, }; - - use datafusion::arrow::array::RecordBatch; - - use datafusion_distributed::{DefaultSessionBuilder, DistributedExt}; - use std::env; use std::fs; + use std::path::Path; use std::sync::Arc; use tokio::sync::OnceCell; - async fn setup() -> Result<(SessionContext, SessionContext, JoinSet<()>)> { - const NUM_WORKERS: usize = 4; - const FILES_PER_TASK: usize = 2; - const CARDINALITY_TASK_COUNT_FACTOR: f64 = 2.0; - - // Make distributed localhost context to run queries - let (mut distributed_ctx, worker_tasks) = - start_localhost_context(NUM_WORKERS, DefaultSessionBuilder).await; - distributed_ctx.set_distributed_files_per_task(FILES_PER_TASK)?; - distributed_ctx - .set_distributed_cardinality_effect_task_scale_factor(CARDINALITY_TASK_COUNT_FACTOR)?; - register_tables_with_options(&distributed_ctx, true).await?; - - // Create single node context to compare results to. - let single_node_ctx = SessionContext::new(); - register_tables_with_options(&single_node_ctx, true).await?; - - Ok((distributed_ctx, single_node_ctx, worker_tasks)) - } + const NUM_WORKERS: usize = 4; + const FILES_PER_TASK: usize = 2; + const CARDINALITY_TASK_COUNT_FACTOR: f64 = 2.0; + const SF: f64 = 1.0; + const PARQUET_PARTITIONS: usize = 4; #[tokio::test] async fn test_tpcds_1() -> Result<()> { @@ -81,6 +65,7 @@ mod tests { } #[tokio::test] + #[ignore = "expected no error but got: Arrow error: Invalid argument error: must either specify a row count or at least one column"] async fn test_tpcds_9() -> Result<()> { test_tpcds_query(9).await } @@ -186,6 +171,7 @@ mod tests { } #[tokio::test] + #[ignore = "Fails with column 'c_last_review_date_sk' not found"] async fn test_tpcds_30() -> Result<()> { test_tpcds_query(30).await } @@ -396,6 +382,10 @@ mod tests { } #[tokio::test] + // For some reason this test takes a ridiculous amount of time to execute. There might be + // nothing wrong with it, and it just might be too heavy. The test passes, but it takes so + // long to execute that it's not worth the time. + #[ignore = "Query takes too long to execute"] async fn test_tpcds_72() -> Result<()> { test_tpcds_query(72).await } @@ -531,59 +521,80 @@ mod tests { } #[tokio::test] + // For some reason this test takes a ridiculous amount of time to execute. There might be + // nothing wrong with it, and it just might be too heavy. The test passes, but it takes so + // long to execute that it's not worth the time. + #[ignore = "Query takes too long to execute"] async fn test_tpcds_99() -> Result<()> { test_tpcds_query(99).await } static INIT_TEST_TPCDS_TABLES: OnceCell<()> = OnceCell::const_new(); - // ensure_tpcds_data initializes the TPCDS data on disk if not already present. - pub async fn ensure_tpcds_data() { - INIT_TEST_TPCDS_TABLES - .get_or_init(|| async { - if !fs::exists(get_data_dir()).unwrap_or(false) { - let scale_factor = - env::var("TPCDS_SCALE_FACTOR").unwrap_or_else(|_| "0.01".to_string()); - generate_tpcds_data(scale_factor.as_str()).unwrap(); - } - }) - .await; - } - async fn run( ctx: &SessionContext, query_sql: &str, - ) -> (Arc, Result>) { + ) -> (Arc, Arc>>) { let df = ctx.sql(query_sql).await.unwrap(); let task_ctx = ctx.task_ctx(); let plan = df.create_physical_plan().await.unwrap(); - (plan.clone(), collect(plan, task_ctx).await) // Collect execution errors, do not unwrap. + (plan.clone(), Arc::new(collect(plan, task_ctx).await)) // Collect execution errors, do not unwrap. } async fn test_tpcds_query(query_id: usize) -> Result<()> { - ensure_tpcds_data().await; - - let query_sql = get_test_tpcds_query(query_id)?; - let (distributed_ctx, single_node_ctx, _handles) = setup().await?; - - let (single_node_physical_plan, single_node_results) = - run(&single_node_ctx, &query_sql).await; - let (distributed_physical_plan, distributed_results) = - run(&distributed_ctx, &query_sql).await; - - let compare_result = tokio::try_join!( - compare_result_set(&distributed_results, &single_node_results), - compare_ordering( - distributed_physical_plan, - single_node_physical_plan, - &distributed_results - ), - ); - assert!( - compare_result.is_ok(), - "Query {query_id} failed: {}", - compare_result.unwrap_err() - ); + let data_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(format!( + "testdata/tpcds/correctness_sf{SF}_partitions{PARQUET_PARTITIONS}" + )); + INIT_TEST_TPCDS_TABLES + .get_or_init(|| async { + if !fs::exists(&data_dir).unwrap_or(false) { + tpcds::generate_tpcds_data(&data_dir, SF, PARQUET_PARTITIONS) + .await + .unwrap(); + } + }) + .await; + + let query_sql = tpcds::get_test_tpcds_query(query_id)?; + // Create a single node context to compare results to. + let s_ctx = SessionContext::new(); + + // Make distributed localhost context to run queries + let (d_ctx, _guard) = start_localhost_context(NUM_WORKERS, DefaultSessionBuilder).await; + let d_ctx = d_ctx + .with_distributed_files_per_task(FILES_PER_TASK)? + .with_distributed_cardinality_effect_task_scale_factor(CARDINALITY_TASK_COUNT_FACTOR)?; + + tpcds::register_tables(&s_ctx, &data_dir).await?; + tpcds::register_tables(&d_ctx, &data_dir).await?; + + let (s_plan, s_results) = run(&s_ctx, &query_sql).await; + let (d_plan, d_results) = run(&d_ctx, &query_sql).await; + + if !d_plan.as_any().is::() { + return plan_err!("Query {query_id} did not get distributed"); + } + let display = display_plan_ascii(d_plan.as_ref(), false); + println!("Query {query_id}:\n{display}"); + + // The comparison functions can be computationally expensive, so we spawn them in tokio + // blocking tasks so that they do not block the tokio runtime. + let compare_result_set = { + let d_results = d_results.clone(); + let s_results = s_results.clone(); + tokio::task::spawn_blocking(move || async move { + compare_result_set(&d_results, &s_results) + }) + }; + let compare_ordering = { + let d_results = d_results.clone(); + tokio::task::spawn_blocking(move || async move { + compare_ordering(d_plan, s_plan, &d_results) + }) + }; + compare_result_set.await.unwrap().await?; + compare_ordering.await.unwrap().await?; + Ok(()) } } diff --git a/tests/tpcds_plans_test.rs b/tests/tpcds_plans_test.rs new file mode 100644 index 00000000..6c201ac7 --- /dev/null +++ b/tests/tpcds_plans_test.rs @@ -0,0 +1,10243 @@ +#[cfg(all(feature = "integration", feature = "tpcds", test))] +mod tests { + use datafusion::error::Result; + use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::test_utils::tpcds; + use datafusion_distributed::{ + DefaultSessionBuilder, DistributedExec, DistributedExt, assert_snapshot, display_plan_ascii, + }; + use std::env; + use std::fs; + use std::path::Path; + use tokio::sync::OnceCell; + + const NUM_WORKERS: usize = 4; + const FILES_PER_TASK: usize = 2; + const CARDINALITY_TASK_COUNT_FACTOR: f64 = 2.0; + const SF: f64 = 1.0; + const PARQUET_PARTITIONS: usize = 4; + + #[tokio::test] + async fn test_tpcds_1() -> Result<()> { + let display = test_tpcds_query(1).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c_customer_id@0 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[c_customer_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ctr_store_sk@0, ctr_store_sk@1)], filter=CAST(ctr_total_return@0 AS Decimal128(30, 15)) > avg(ctr2.ctr_total_return) * Float64(1.2)@1, projection=[c_customer_id@2] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ctr_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@2)], projection=[ctr_store_sk@1, ctr_total_return@2, c_customer_id@4] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ctr_store_sk@1)], projection=[ctr_customer_sk@2, ctr_store_sk@3, ctr_total_return@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[sr_customer_sk@0 as ctr_customer_sk, sr_store_sk@1 as ctr_store_sk, sum(store_returns.sr_return_amt)@2 as ctr_total_return] + │ AggregateExec: mode=FinalPartitioned, gby=[sr_customer_sk@0 as sr_customer_sk, sr_store_sk@1 as sr_store_sk], aggr=[sum(store_returns.sr_return_amt)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_customer_sk@0, sr_store_sk@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[sr_customer_sk@0 as sr_customer_sk, sr_store_sk@1 as sr_store_sk], aggr=[sum(store_returns.sr_return_amt)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, sr_returned_date_sk@0)], projection=[sr_customer_sk@3, sr_store_sk@4, sr_return_amt@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_customer_sk, sr_store_sk, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[CAST(CAST(avg(ctr2.ctr_total_return)@1 AS Float64) * 1.2 AS Decimal128(30, 15)) as avg(ctr2.ctr_total_return) * Float64(1.2), ctr_store_sk@0 as ctr_store_sk] + │ AggregateExec: mode=FinalPartitioned, gby=[ctr_store_sk@0 as ctr_store_sk], aggr=[avg(ctr2.ctr_total_return)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ctr_store_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ctr_store_sk@0 as ctr_store_sk], aggr=[avg(ctr2.ctr_total_return)] + │ ProjectionExec: expr=[sr_store_sk@1 as ctr_store_sk, sum(store_returns.sr_return_amt)@2 as ctr_total_return] + │ AggregateExec: mode=FinalPartitioned, gby=[sr_customer_sk@0 as sr_customer_sk, sr_store_sk@1 as sr_store_sk], aggr=[sum(store_returns.sr_return_amt)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_customer_sk@0, sr_store_sk@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[sr_customer_sk@0 as sr_customer_sk, sr_store_sk@1 as sr_store_sk], aggr=[sum(store_returns.sr_return_amt)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, sr_returned_date_sk@0)], projection=[sr_customer_sk@3, sr_store_sk@4, sr_return_amt@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_customer_sk, sr_store_sk, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@1 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_tpcds_2() -> Result<()> { + let display = test_tpcds_query(2).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [d_week_seq1@0 ASC] + │ SortExec: expr=[d_week_seq1@0 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[d_week_seq1@0 as d_week_seq1, round(CAST(sun_sales1@1 / sun_sales2@8 AS Float64), 2) as r1, round(CAST(mon_sales1@2 / mon_sales2@9 AS Float64), 2) as r2, round(CAST(tue_sales1@3 / tue_sales2@10 AS Float64), 2) as r3, round(CAST(wed_sales1@4 / wed_sales2@11 AS Float64), 2) as r4, round(CAST(thu_sales1@5 / thu_sales2@12 AS Float64), 2) as r5, round(CAST(fri_sales1@6 / fri_sales2@13 AS Float64), 2) as r6, round(CAST(sat_sales1@7 / sat_sales2@14 AS Float64), 2) as round(y.sat_sales1 / z.sat_sales2,Int64(2))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(y.d_week_seq1 AS Int64)@8, z.d_week_seq2 - Int64(53)@8)], projection=[d_week_seq1@0, sun_sales1@1, mon_sales1@2, tue_sales1@3, wed_sales1@4, thu_sales1@5, fri_sales1@6, sat_sales1@7, sun_sales2@10, mon_sales2@11, tue_sales2@12, wed_sales2@13, thu_sales2@14, fri_sales2@15, sat_sales2@16] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq1, sun_sales@1 as sun_sales1, mon_sales@2 as mon_sales1, tue_sales@3 as tue_sales1, wed_sales@4 as wed_sales1, thu_sales@5 as thu_sales1, fri_sales@6 as fri_sales1, sat_sales@7 as sat_sales1, CAST(d_week_seq@0 AS Int64) as CAST(y.d_week_seq1 AS Int64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@0)], projection=[d_week_seq@1, sun_sales@2, mon_sales@3, tue_sales@4, wed_sales@5, thu_sales@6, fri_sales@7, sat_sales@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq, sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END)@1 as sun_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END)@2 as mon_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END)@3 as tue_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END)@4 as wed_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END)@5 as thu_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END)@6 as fri_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)@7 as sat_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[d_week_seq@0 as d_week_seq], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq2, sun_sales@1 as sun_sales2, mon_sales@2 as mon_sales2, tue_sales@3 as tue_sales2, wed_sales@4 as wed_sales2, thu_sales@5 as thu_sales2, fri_sales@6 as fri_sales2, sat_sales@7 as sat_sales2, CAST(d_week_seq@0 AS Int64) - 53 as z.d_week_seq2 - Int64(53)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@0)], projection=[d_week_seq@1, sun_sales@2, mon_sales@3, tue_sales@4, wed_sales@5, thu_sales@6, fri_sales@7, sat_sales@8] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq, sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END)@1 as sun_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END)@2 as mon_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END)@3 as tue_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END)@4 as wed_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END)@5 as thu_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END)@6 as fri_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)@7 as sat_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[d_week_seq@0 as d_week_seq], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)] + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001, projection=[d_week_seq@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([d_week_seq@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_week_seq@1 as d_week_seq], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)] + │ ProjectionExec: expr=[sales_price@2 as sales_price, d_week_seq@0 as d_week_seq, d_day_name@1 as d_day_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, sold_date_sk@0)], projection=[d_week_seq@1, d_day_name@2, sales_price@5] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 6), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_week_seq@1 as d_week_seq, d_day_name@2 as d_day_name, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sold_date_sk@0], 6), input_partitions=6 + │ UnionExec + │ ProjectionExec: expr=[ws_sold_date_sk@0 as sold_date_sk, ws_ext_sales_price@1 as sales_price] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_ext_sales_price], file_type=parquet + │ ProjectionExec: expr=[cs_sold_date_sk@0 as sold_date_sk, cs_ext_sales_price@1 as sales_price] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_ext_sales_price], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002, projection=[d_week_seq@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([d_week_seq@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_week_seq@1 as d_week_seq], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)] + │ ProjectionExec: expr=[sales_price@2 as sales_price, d_week_seq@0 as d_week_seq, d_day_name@1 as d_day_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, sold_date_sk@0)], projection=[d_week_seq@1, d_day_name@2, sales_price@5] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 6), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_week_seq@1 as d_week_seq, d_day_name@2 as d_day_name, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sold_date_sk@0], 6), input_partitions=6 + │ UnionExec + │ ProjectionExec: expr=[ws_sold_date_sk@0 as sold_date_sk, ws_ext_sales_price@1 as sales_price] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_ext_sales_price], file_type=parquet + │ ProjectionExec: expr=[cs_sold_date_sk@0 as sold_date_sk, cs_ext_sales_price@1 as sales_price] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_ext_sales_price], file_type=parquet + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_3() -> Result<()> { + let display = test_tpcds_query(3).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [d_year@0 ASC NULLS LAST, sum_agg@3 DESC, brand_id@1 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[d_year@0 ASC NULLS LAST, sum_agg@3 DESC, brand_id@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[d_year@0 as d_year, i_brand_id@2 as brand_id, i_brand@1 as brand, sum(store_sales.ss_ext_sales_price)@3 as sum_agg] + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand@1 as i_brand, i_brand_id@2 as i_brand_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([d_year@0, i_brand@1, i_brand_id@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand@3 as i_brand, i_brand_id@2 as i_brand_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@1, i_item_sk@0)], projection=[d_year@0, ss_ext_sales_price@2, i_brand_id@4, i_brand@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(dt.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_manufact_id@3 = 128, projection=[i_item_sk@0, i_brand_id@1, i_brand@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manufact_id], file_type=parquet, predicate=i_manufact_id@3 = 128 AND DynamicFilter [ empty ], pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 128 AND 128 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (128)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(dt.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_moy@2 = 11, projection=[d_date_sk@0, d_year@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 11, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1, required_guarantees=[d_moy in (11)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_4() -> Result<()> { + let display = test_tpcds_query(4).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [customer_id@0 ASC, customer_first_name@1 ASC, customer_last_name@2 ASC, customer_preferred_cust_flag@3 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[customer_id@0 ASC, customer_first_name@1 ASC, customer_last_name@2 ASC, customer_preferred_cust_flag@3 ASC], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], filter=CASE WHEN year_total@0 > Some(0),24,6 THEN year_total@1 / year_total@0 END > CASE WHEN year_total@2 > Some(0),24,6 THEN year_total@3 / year_total@2 END, projection=[customer_id@1, customer_first_name@2, customer_last_name@3, customer_preferred_cust_flag@4] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[customer_id@1 as customer_id, customer_id@2 as customer_id, customer_first_name@3 as customer_first_name, customer_last_name@4 as customer_last_name, customer_preferred_cust_flag@5 as customer_preferred_cust_flag, year_total@6 as year_total, year_total@7 as year_total, year_total@0 as year_total] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], projection=[year_total@1, customer_id@2, customer_id@3, customer_first_name@4, customer_last_name@5, customer_preferred_cust_flag@6, year_total@7, year_total@8] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))@1 as year_total] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))@1 > Some(0),24,6 + │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))@8 as sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, ws_ext_discount_amt@8 as ws_ext_discount_amt, ws_ext_sales_price@9 as ws_ext_sales_price, ws_ext_wholesale_cost@10 as ws_ext_wholesale_cost, ws_ext_list_price@11 as ws_ext_list_price, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@7)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, c_preferred_cust_flag@6, c_birth_country@7, c_login@8, c_email_address@9, ws_ext_discount_amt@11, ws_ext_sales_price@12, ws_ext_wholesale_cost@13, ws_ext_list_price@14] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ws_sold_date_sk@9, ws_ext_discount_amt@11, ws_ext_sales_price@12, ws_ext_wholesale_cost@13, ws_ext_list_price@14] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], filter=CASE WHEN year_total@2 > Some(0),24,6 THEN year_total@3 / year_total@2 END > CASE WHEN year_total@0 > Some(0),24,6 THEN year_total@1 / year_total@0 END, projection=[customer_id@0, customer_id@2, customer_first_name@3, customer_last_name@4, customer_preferred_cust_flag@5, year_total@7, year_total@9] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[customer_id@1 as customer_id, year_total@2 as year_total, customer_id@3 as customer_id, customer_first_name@4 as customer_first_name, customer_last_name@5 as customer_last_name, customer_preferred_cust_flag@6 as customer_preferred_cust_flag, year_total@7 as year_total, year_total@0 as year_total] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], projection=[year_total@1, customer_id@2, year_total@3, customer_id@4, customer_first_name@5, customer_last_name@6, customer_preferred_cust_flag@7, year_total@8] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))@1 as year_total] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))@1 > Some(0),24,6 + │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))@8 as sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, cs_ext_discount_amt@8 as cs_ext_discount_amt, cs_ext_sales_price@9 as cs_ext_sales_price, cs_ext_wholesale_cost@10 as cs_ext_wholesale_cost, cs_ext_list_price@11 as cs_ext_list_price, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@7)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, c_preferred_cust_flag@6, c_birth_country@7, c_login@8, c_email_address@9, cs_ext_discount_amt@11, cs_ext_sales_price@12, cs_ext_wholesale_cost@13, cs_ext_list_price@14] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, cs_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, cs_sold_date_sk@9, cs_ext_discount_amt@11, cs_ext_sales_price@12, cs_ext_wholesale_cost@13, cs_ext_list_price@14] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))@1 as year_total] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))@1 > Some(0),24,6 + │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))@8 as sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, ss_ext_discount_amt@8 as ss_ext_discount_amt, ss_ext_sales_price@9 as ss_ext_sales_price, ss_ext_wholesale_cost@10 as ss_ext_wholesale_cost, ss_ext_list_price@11 as ss_ext_list_price, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@7)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, c_preferred_cust_flag@6, c_birth_country@7, c_login@8, c_email_address@9, ss_ext_discount_amt@11, ss_ext_sales_price@12, ss_ext_wholesale_cost@13, ss_ext_list_price@14] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ss_sold_date_sk@9, ss_ext_discount_amt@11, ss_ext_sales_price@12, ss_ext_wholesale_cost@13, ss_ext_list_price@14] + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, c_first_name@1 as customer_first_name, c_last_name@2 as customer_last_name, c_preferred_cust_flag@3 as customer_preferred_cust_flag, sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))@8 as year_total] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, ss_ext_discount_amt@8 as ss_ext_discount_amt, ss_ext_sales_price@9 as ss_ext_sales_price, ss_ext_wholesale_cost@10 as ss_ext_wholesale_cost, ss_ext_list_price@11 as ss_ext_list_price, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@7)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, c_preferred_cust_flag@6, c_birth_country@7, c_login@8, c_email_address@9, ss_ext_discount_amt@11, ss_ext_sales_price@12, ss_ext_wholesale_cost@13, ss_ext_list_price@14] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ss_sold_date_sk@9, ss_ext_discount_amt@11, ss_ext_sales_price@12, ss_ext_wholesale_cost@13, ss_ext_list_price@14] + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))@8 as year_total] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, cs_ext_discount_amt@8 as cs_ext_discount_amt, cs_ext_sales_price@9 as cs_ext_sales_price, cs_ext_wholesale_cost@10 as cs_ext_wholesale_cost, cs_ext_list_price@11 as cs_ext_list_price, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@7)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, c_preferred_cust_flag@6, c_birth_country@7, c_login@8, c_email_address@9, cs_ext_discount_amt@11, cs_ext_sales_price@12, cs_ext_wholesale_cost@13, cs_ext_list_price@14] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, cs_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, cs_sold_date_sk@9, cs_ext_discount_amt@11, cs_ext_sales_price@12, cs_ext_wholesale_cost@13, cs_ext_list_price@14] + │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))@8 as year_total] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, ws_ext_discount_amt@8 as ws_ext_discount_amt, ws_ext_sales_price@9 as ws_ext_sales_price, ws_ext_wholesale_cost@10 as ws_ext_wholesale_cost, ws_ext_list_price@11 as ws_ext_list_price, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@7)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, c_preferred_cust_flag@6, c_birth_country@7, c_login@8, c_email_address@9, ws_ext_discount_amt@11, ws_ext_sales_price@12, ws_ext_wholesale_cost@13, ws_ext_list_price@14] + │ CoalescePartitionsExec + │ [Stage 16] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ws_sold_date_sk@9, ws_ext_discount_amt@11, ws_ext_sales_price@12, ws_ext_wholesale_cost@13, ws_ext_list_price@14] + │ [Stage 17] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 18] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@8], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@8], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@8], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@8], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@8], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 17 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@8], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 18 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_5() -> Result<()> { + let display = test_tpcds_query(5).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [channel@0 ASC, id@1 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, id@1 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[channel@0 as channel, id@1 as id, sum(x.sales)@3 as sales, sum(x.returns_)@4 as returns_, sum(x.profit)@5 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, id@1 as id, __grouping_id@2 as __grouping_id], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([channel@0, id@1, __grouping_id@2], 3), input_partitions=9 + │ AggregateExec: mode=Partial, gby=[(NULL as channel, NULL as id), (channel@0 as channel, NULL as id), (channel@0 as channel, id@1 as id)], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] + │ UnionExec + │ ProjectionExec: expr=[store channel as channel, concat(store, s_store_id@0) as id, sales@1 as sales, returns_@3 as returns_, profit@2 - profit_loss@4 as profit] + │ ProjectionExec: expr=[s_store_id@0 as s_store_id, sum(salesreturns.sales_price)@1 as sales, sum(salesreturns.profit)@2 as profit, sum(salesreturns.return_amt)@3 as returns_, sum(salesreturns.net_loss)@4 as profit_loss] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_id@0 as s_store_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([s_store_id@0], 3), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[s_store_id@4 as s_store_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] + │ ProjectionExec: expr=[sales_price@1 as sales_price, profit@2 as profit, return_amt@3 as return_amt, net_loss@4 as net_loss, s_store_id@0 as s_store_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, store_sk@0)], projection=[s_store_id@1, sales_price@4, profit@5, return_amt@6, net_loss@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, date_sk@1)], projection=[store_sk@2, sales_price@4, profit@5, return_amt@6, net_loss@7] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ UnionExec + │ ProjectionExec: expr=[ss_store_sk@1 as store_sk, ss_sold_date_sk@0 as date_sk, ss_ext_sales_price@2 as sales_price, CAST(ss_net_profit@3 AS Decimal128(7, 2)) as profit, Some(0),7,2 as return_amt, Some(0),7,2 as net_loss] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet + │ ProjectionExec: expr=[sr_store_sk@1 as store_sk, sr_returned_date_sk@0 as date_sk, Some(0),7,2 as sales_price, Some(0),7,2 as profit, sr_return_amt@2 as return_amt, CAST(sr_net_loss@3 AS Decimal128(7, 2)) as net_loss] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_store_sk, sr_return_amt, sr_net_loss], file_type=parquet + │ ProjectionExec: expr=[catalog channel as channel, concat(catalog_page, cp_catalog_page_id@0) as id, sales@1 as sales, returns_@3 as returns_, profit@2 - profit_loss@4 as profit] + │ ProjectionExec: expr=[cp_catalog_page_id@0 as cp_catalog_page_id, sum(salesreturns.sales_price)@1 as sales, sum(salesreturns.profit)@2 as profit, sum(salesreturns.return_amt)@3 as returns_, sum(salesreturns.net_loss)@4 as profit_loss] + │ AggregateExec: mode=FinalPartitioned, gby=[cp_catalog_page_id@0 as cp_catalog_page_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[web channel as channel, concat(web_site, web_site_id@0) as id, sales@1 as sales, returns_@3 as returns_, profit@2 - profit_loss@4 as profit] + │ ProjectionExec: expr=[web_site_id@0 as web_site_id, sum(salesreturns.sales_price)@1 as sales, sum(salesreturns.profit)@2 as profit, sum(salesreturns.return_amt)@3 as returns_, sum(salesreturns.net_loss)@4 as profit_loss] + │ AggregateExec: mode=FinalPartitioned, gby=[web_site_id@0 as web_site_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([web_site_id@0], 3), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[web_site_id@4 as web_site_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] + │ ProjectionExec: expr=[sales_price@1 as sales_price, profit@2 as profit, return_amt@3 as return_amt, net_loss@4 as net_loss, web_site_id@0 as web_site_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_site.web_site_sk AS Float64)@2, wsr_web_site_sk@0)], projection=[web_site_id@1, sales_price@4, profit@5, return_amt@6, net_loss@7] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, date_sk@1)], projection=[wsr_web_site_sk@2, sales_price@4, profit@5, return_amt@6, net_loss@7] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ UnionExec + │ ProjectionExec: expr=[ws_web_site_sk@1 as wsr_web_site_sk, ws_sold_date_sk@0 as date_sk, ws_ext_sales_price@2 as sales_price, ws_net_profit@3 as profit, Some(0),7,2 as return_amt, Some(0),7,2 as net_loss] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_web_site_sk, ws_ext_sales_price, ws_net_profit], file_type=parquet + │ ProjectionExec: expr=[ws_web_site_sk@6 as wsr_web_site_sk, wr_returned_date_sk@0 as date_sk, Some(0),7,2 as sales_price, Some(0),7,2 as profit, wr_return_amt@3 as return_amt, wr_net_loss@4 as net_loss] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(wr_item_sk@1, ws_item_sk@0), (wr_order_number@2, ws_order_number@2)] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_id@1 as s_store_id, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cp_catalog_page_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cp_catalog_page_id@4 as cp_catalog_page_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(page_sk@0, CAST(catalog_page.cp_catalog_page_sk AS Float64)@2)], projection=[sales_price@1, profit@2, return_amt@3, net_loss@4, cp_catalog_page_id@6] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=1 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([page_sk@0], 6), input_partitions=6 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, date_sk@1)], projection=[page_sk@2, sales_price@4, profit@5, return_amt@6, net_loss@7] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ UnionExec + │ ProjectionExec: expr=[cs_catalog_page_sk@1 as page_sk, cs_sold_date_sk@0 as date_sk, cs_ext_sales_price@2 as sales_price, cs_net_profit@3 as profit, Some(0),7,2 as return_amt, Some(0),7,2 as net_loss] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_catalog_page_sk, cs_ext_sales_price, cs_net_profit], file_type=parquet + │ ProjectionExec: expr=[cr_catalog_page_sk@1 as page_sk, CAST(cr_returned_date_sk@0 AS Float64) as date_sk, Some(0),7,2 as sales_price, Some(0),7,2 as profit, cr_return_amount@2 as return_amt, cr_net_loss@3 as net_loss] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_catalog_page_sk, cr_return_amount, cr_net_loss], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(catalog_page.cp_catalog_page_sk AS Float64)@2], 6), input_partitions=3 + │ ProjectionExec: expr=[cp_catalog_page_sk@0 as cp_catalog_page_sk, cp_catalog_page_id@1 as cp_catalog_page_id, CAST(cp_catalog_page_sk@0 AS Float64) as CAST(catalog_page.cp_catalog_page_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-3.parquet]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[web_site_sk@0 as web_site_sk, web_site_id@1 as web_site_id, CAST(web_site_sk@0 AS Float64) as CAST(web_site.web_site_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-3.parquet]]}, projection=[web_site_sk, web_site_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([wr_item_sk@1, wr_order_number@2], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_item_sk@0, ws_order_number@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_item_sk, ws_web_site_sk, ws_order_number], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_6() -> Result<()> { + let display = test_tpcds_query(6).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[state@0 as state, cnt@1 as cnt] + │ SortPreservingMergeExec: [cnt@1 ASC, ca_state@2 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[cnt@1 ASC, state@0 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[ca_state@0 as state, count(Int64(1))@1 as cnt, ca_state@0 as ca_state] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: count(Int64(1))@1 >= 10 + │ AggregateExec: mode=FinalPartitioned, gby=[ca_state@0 as ca_state], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ca_state@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_state@0 as ca_state], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CAST(i_current_price@1 AS Decimal128(30, 15)) > CAST(1.2 * avg(j.i_current_price)@2 AS Decimal128(30, 15)), projection=[ca_state@0] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(i_category@2, i_category@1)], projection=[ca_state@0, i_current_price@1, avg(j.i_current_price)@3] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_month_seq@0, d_month_seq@1)], projection=[ca_state@1, i_current_price@3, i_category@4] + │ CoalescePartitionsExec + │ AggregateExec: mode=FinalPartitioned, gby=[d_month_seq@0 as d_month_seq], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@1, i_item_sk@0)], projection=[ca_state@0, d_month_seq@2, i_current_price@4, i_category@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_sold_date_sk@1, CAST(d.d_date_sk AS Float64)@2)], projection=[ca_state@0, ss_item_sk@2, d_month_seq@4] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(c.c_customer_sk AS Float64)@2, ss_customer_sk@2)], projection=[ca_state@0, ss_sold_date_sk@3, ss_item_sk@4] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_month_seq@1 as d_month_seq, CAST(d_date_sk@0 AS Float64) as CAST(d.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=DynamicFilter [ empty ] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price, i_category], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[CAST(avg(j.i_current_price)@1 AS Float64) as avg(j.i_current_price), i_category@0 as i_category] + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category], aggr=[avg(j.i_current_price)] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([d_month_seq@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_month_seq@0 as d_month_seq], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 1, projection=[d_month_seq@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_moy in (1), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ ProjectionExec: expr=[ca_state@0 as ca_state, c_customer_sk@1 as c_customer_sk, CAST(c_customer_sk@1 AS Float64) as CAST(c.c_customer_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@1)], projection=[ca_state@1, c_customer_sk@2] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ca_address_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_current_addr_sk@1], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@1 as i_category], aggr=[avg(j.i_current_price)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_current_price, i_category], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_7() -> Result<()> { + let display = test_tpcds_query(7).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, avg(store_sales.ss_quantity)@1 as agg1, avg(store_sales.ss_list_price)@2 as agg2, avg(store_sales.ss_coupon_amt)@3 as agg3, avg(store_sales.ss_sales_price)@4 as agg4] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[avg(store_sales.ss_quantity), avg(store_sales.ss_list_price), avg(store_sales.ss_coupon_amt), avg(store_sales.ss_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@4 as i_item_id], aggr=[avg(store_sales.ss_quantity), avg(store_sales.ss_list_price), avg(store_sales.ss_coupon_amt), avg(store_sales.ss_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(promotion.p_promo_sk AS Float64)@1, ss_promo_sk@0)], projection=[ss_quantity@3, ss_list_price@4, ss_sales_price@5, ss_coupon_amt@6, i_item_id@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_promo_sk@1, ss_quantity@2, ss_list_price@3, ss_sales_price@4, ss_coupon_amt@5, i_item_id@7] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_promo_sk@4, ss_quantity@5, ss_list_price@6, ss_sales_price@7, ss_coupon_amt@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, ss_cdemo_sk@2)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_promo_sk@5, ss_quantity@6, ss_list_price@7, ss_sales_price@8, ss_coupon_amt@9] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: p_channel_email@1 = N OR p_channel_event@2 = N, projection=[p_promo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, p_channel_email, p_channel_event], file_type=parquet, predicate=p_channel_email@1 = N OR p_channel_event@2 = N, pruning_predicate=p_channel_email_null_count@2 != row_count@3 AND p_channel_email_min@0 <= N AND N <= p_channel_email_max@1 OR p_channel_event_null_count@6 != row_count@3 AND p_channel_event_min@4 <= N AND N <= p_channel_event_max@5, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=3 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_promo_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_8() -> Result<()> { + let display = test_tpcds_query(8).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [s_store_name@0 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[s_store_name@0 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_name@0 as s_store_name], aggr=[sum(store_sales.ss_net_profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([s_store_name@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[s_store_name@1 as s_store_name], aggr=[sum(store_sales.ss_net_profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(substr(store.s_zip,Int64(1),Int64(2))@3, substr(v1.ca_zip,Int64(1),Int64(2))@1)], projection=[ss_net_profit@0, s_store_name@1] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_net_profit@0 as ss_net_profit, s_store_name@1 as s_store_name, s_zip@2 as s_zip, substr(s_zip@2, 1, 2) as substr(store.s_zip,Int64(1),Int64(2))] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_store_sk@0, CAST(store.s_store_sk AS Float64)@3)], projection=[ss_net_profit@1, s_store_name@3, s_zip@4] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_store_sk@3, ss_net_profit@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_name@1 as s_store_name, s_zip@2 as s_zip, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_zip], file_type=parquet + │ ProjectionExec: expr=[ca_zip@0 as ca_zip, substr(ca_zip@0, 1, 2) as substr(v1.ca_zip,Int64(1),Int64(2))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(ca_zip@0, ca_zip@0)], NullsEqual: true + │ CoalescePartitionsExec + │ AggregateExec: mode=FinalPartitioned, gby=[ca_zip@0 as ca_zip], aggr=[] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[substr(ca_zip@0, 1, 5) as ca_zip] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: count(Int64(1))@1 > 10, projection=[ca_zip@0] + │ AggregateExec: mode=FinalPartitioned, gby=[ca_zip@0 as ca_zip], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ca_zip@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_zip@0 as ca_zip], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@0, ca_address_sk@0)], projection=[ca_zip@2] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_qoy@2 = 2 AND d_year@1 = 1998, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@2 = 2 AND d_year@1 = 1998, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ca_zip@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_zip@0 as ca_zip], aggr=[] + │ ProjectionExec: expr=[substr(ca_zip@0, 1, 5) as ca_zip] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: substr(ca_zip@0, 1, 5) IN ([24128, 76232, 65084, 87816, 83926, 77556, 20548, 26231, 43848, 15126, 91137, 61265, 98294, 25782, 17920, 18426, 98235, 40081, 84093, 28577, 55565, 17183, 54601, 67897, 22752, 86284, 18376, 38607, 45200, 21756, 29741, 96765, 23932, 89360, 29839, 25989, 28898, 91068, 72550, 10390, 18845, 47770, 82636, 41367, 76638, 86198, 81312, 37126, 39192, 88424, 72175, 81426, 53672, 10445, 42666, 66864, 66708, 41248, 48583, 82276, 18842, 78890, 49448, 14089, 38122, 34425, 79077, 19849, 43285, 39861, 66162, 77610, 13695, 99543, 83444, 83041, 12305, 57665, 68341, 25003, 57834, 62878, 49130, 81096, 18840, 27700, 23470, 50412, 21195, 16021, 76107, 71954, 68309, 18119, 98359, 64544, 10336, 86379, 27068, 39736, 98569, 28915, 24206, 56529, 57647, 54917, 42961, 91110, 63981, 14922, 36420, 23006, 67467, 32754, 30903, 20260, 31671, 51798, 72325, 85816, 68621, 13955, 36446, 41766, 68806, 16725, 15146, 22744, 35850, 88086, 51649, 18270, 52867, 39972, 96976, 63792, 11376, 94898, 13595, 10516, 90225, 58943, 39371, 94945, 28587, 96576, 57855, 28488, 26105, 83933, 25858, 34322, 44438, 73171, 30122, 34102, 22685, 71256, 78451, 54364, 13354, 45375, 40558, 56458, 28286, 45266, 47305, 69399, 83921, 26233, 11101, 15371, 69913, 35942, 15882, 25631, 24610, 44165, 99076, 33786, 70738, 26653, 14328, 72305, 62496, 22152, 10144, 64147, 48425, 14663, 21076, 18799, 30450, 63089, 81019, 68893, 24996, 51200, 51211, 45692, 92712, 70466, 79994, 22437, 25280, 38935, 71791, 73134, 56571, 14060, 19505, 72425, 56575, 74351, 68786, 51650, 20004, 18383, 76614, 11634, 18906, 15765, 41368, 73241, 76698, 78567, 97189, 28545, 76231, 75691, 22246, 51061, 90578, 56691, 68014, 51103, 94167, 57047, 14867, 73520, 15734, 63435, 25733, 35474, 24676, 94627, 53535, 17879, 15559, 53268, 59166, 11928, 59402, 33282, 45721, 43933, 68101, 33515, 36634, 71286, 19736, 58058, 55253, 67473, 41918, 19515, 36495, 19430, 22351, 77191, 91393, 49156, 50298, 87501, 18652, 53179, 18767, 63193, 23968, 65164, 68880, 21286, 72823, 58470, 67301, 13394, 31016, 70372, 67030, 40604, 24317, 45748, 39127, 26065, 77721, 31029, 31880, 60576, 24671, 45549, 13376, 50016, 33123, 19769, 22927, 97789, 46081, 72151, 15723, 46136, 51949, 68100, 96888, 64528, 14171, 79777, 28709, 11489, 25103, 32213, 78668, 22245, 15798, 27156, 37930, 62971, 21337, 51622, 67853, 10567, 38415, 15455, 58263, 42029, 60279, 37125, 56240, 88190, 50308, 26859, 64457, 89091, 82136, 62377, 36233, 63837, 58078, 17043, 30010, 60099, 28810, 98025, 29178, 87343, 73273, 30469, 64034, 39516, 86057, 21309, 90257, 67875, 40162, 11356, 73650, 61810, 72013, 30431, 22461, 19512, 13375, 55307, 30625, 83849, 68908, 26689, 96451, 38193, 46820, 88885, 84935, 69035, 83144, 47537, 56616, 94983, 48033, 69952, 25486, 61547, 27385, 61860, 58048, 56910, 16807, 17871, 35258, 31387, 35458, 35576]) + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_zip], file_type=parquet, predicate=substr(ca_zip@0, 1, 5) IN ([24128, 76232, 65084, 87816, 83926, 77556, 20548, 26231, 43848, 15126, 91137, 61265, 98294, 25782, 17920, 18426, 98235, 40081, 84093, 28577, 55565, 17183, 54601, 67897, 22752, 86284, 18376, 38607, 45200, 21756, 29741, 96765, 23932, 89360, 29839, 25989, 28898, 91068, 72550, 10390, 18845, 47770, 82636, 41367, 76638, 86198, 81312, 37126, 39192, 88424, 72175, 81426, 53672, 10445, 42666, 66864, 66708, 41248, 48583, 82276, 18842, 78890, 49448, 14089, 38122, 34425, 79077, 19849, 43285, 39861, 66162, 77610, 13695, 99543, 83444, 83041, 12305, 57665, 68341, 25003, 57834, 62878, 49130, 81096, 18840, 27700, 23470, 50412, 21195, 16021, 76107, 71954, 68309, 18119, 98359, 64544, 10336, 86379, 27068, 39736, 98569, 28915, 24206, 56529, 57647, 54917, 42961, 91110, 63981, 14922, 36420, 23006, 67467, 32754, 30903, 20260, 31671, 51798, 72325, 85816, 68621, 13955, 36446, 41766, 68806, 16725, 15146, 22744, 35850, 88086, 51649, 18270, 52867, 39972, 96976, 63792, 11376, 94898, 13595, 10516, 90225, 58943, 39371, 94945, 28587, 96576, 57855, 28488, 26105, 83933, 25858, 34322, 44438, 73171, 30122, 34102, 22685, 71256, 78451, 54364, 13354, 45375, 40558, 56458, 28286, 45266, 47305, 69399, 83921, 26233, 11101, 15371, 69913, 35942, 15882, 25631, 24610, 44165, 99076, 33786, 70738, 26653, 14328, 72305, 62496, 22152, 10144, 64147, 48425, 14663, 21076, 18799, 30450, 63089, 81019, 68893, 24996, 51200, 51211, 45692, 92712, 70466, 79994, 22437, 25280, 38935, 71791, 73134, 56571, 14060, 19505, 72425, 56575, 74351, 68786, 51650, 20004, 18383, 76614, 11634, 18906, 15765, 41368, 73241, 76698, 78567, 97189, 28545, 76231, 75691, 22246, 51061, 90578, 56691, 68014, 51103, 94167, 57047, 14867, 73520, 15734, 63435, 25733, 35474, 24676, 94627, 53535, 17879, 15559, 53268, 59166, 11928, 59402, 33282, 45721, 43933, 68101, 33515, 36634, 71286, 19736, 58058, 55253, 67473, 41918, 19515, 36495, 19430, 22351, 77191, 91393, 49156, 50298, 87501, 18652, 53179, 18767, 63193, 23968, 65164, 68880, 21286, 72823, 58470, 67301, 13394, 31016, 70372, 67030, 40604, 24317, 45748, 39127, 26065, 77721, 31029, 31880, 60576, 24671, 45549, 13376, 50016, 33123, 19769, 22927, 97789, 46081, 72151, 15723, 46136, 51949, 68100, 96888, 64528, 14171, 79777, 28709, 11489, 25103, 32213, 78668, 22245, 15798, 27156, 37930, 62971, 21337, 51622, 67853, 10567, 38415, 15455, 58263, 42029, 60279, 37125, 56240, 88190, 50308, 26859, 64457, 89091, 82136, 62377, 36233, 63837, 58078, 17043, 30010, 60099, 28810, 98025, 29178, 87343, 73273, 30469, 64034, 39516, 86057, 21309, 90257, 67875, 40162, 11356, 73650, 61810, 72013, 30431, 22461, 19512, 13375, 55307, 30625, 83849, 68908, 26689, 96451, 38193, 46820, 88885, 84935, 69035, 83144, 47537, 56616, 94983, 48033, 69952, 25486, 61547, 27385, 61860, 58048, 56910, 16807, 17871, 35258, 31387, 35458, 35576]) + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: c_preferred_cust_flag@1 = Y, projection=[c_current_addr_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_current_addr_sk, c_preferred_cust_flag], file_type=parquet, predicate=c_preferred_cust_flag@1 = Y, pruning_predicate=c_preferred_cust_flag_null_count@2 != row_count@3 AND c_preferred_cust_flag_min@0 <= Y AND Y <= c_preferred_cust_flag_max@1, required_guarantees=[c_preferred_cust_flag in (Y)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_9() -> Result<()> { + let display = test_tpcds_query(9).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[CASE WHEN count(*)@0 > 74129 THEN avg(store_sales.ss_ext_discount_amt)@1 ELSE avg(store_sales.ss_net_paid)@2 END as bucket1, CASE WHEN count(*)@3 > 122840 THEN avg(store_sales.ss_ext_discount_amt)@4 ELSE avg(store_sales.ss_net_paid)@5 END as bucket2, CASE WHEN count(*)@6 > 56580 THEN avg(store_sales.ss_ext_discount_amt)@7 ELSE avg(store_sales.ss_net_paid)@8 END as bucket3, CASE WHEN count(*)@9 > 10097 THEN avg(store_sales.ss_ext_discount_amt)@10 ELSE avg(store_sales.ss_net_paid)@11 END as bucket4, CASE WHEN count(*)@12 > 165306 THEN avg(store_sales.ss_ext_discount_amt)@13 ELSE avg(store_sales.ss_net_paid)@14 END as bucket5] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1 + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_net_paid)] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_net_paid)] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_net_paid)] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_net_paid)] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 14] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] + │ CoalescePartitionsExec + │ [Stage 15] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_net_paid)] + │ CoalescePartitionsExec + │ [Stage 16] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: r_reason_sk@0 = 1 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/reason/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-3.parquet]]}, projection=[r_reason_sk], file_type=parquet, predicate=r_reason_sk@0 = 1, pruning_predicate=r_reason_sk_null_count@2 != row_count@3 AND r_reason_sk_min@0 <= 1 AND 1 <= r_reason_sk_max@1, required_guarantees=[r_reason_sk in (1)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 1 AND ss_quantity@0 <= 20 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity], file_type=parquet, predicate=ss_quantity@0 >= 1 AND ss_quantity@0 <= 20, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 1 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 20, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 1 AND ss_quantity@0 <= 20, projection=[ss_ext_discount_amt@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_ext_discount_amt], file_type=parquet, predicate=ss_quantity@0 >= 1 AND ss_quantity@0 <= 20, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 1 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 20, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_net_paid)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 1 AND ss_quantity@0 <= 20, projection=[ss_net_paid@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_net_paid], file_type=parquet, predicate=ss_quantity@0 >= 1 AND ss_quantity@0 <= 20, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 1 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 20, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 21 AND ss_quantity@0 <= 40 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity], file_type=parquet, predicate=ss_quantity@0 >= 21 AND ss_quantity@0 <= 40, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 21 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 40, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 21 AND ss_quantity@0 <= 40, projection=[ss_ext_discount_amt@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_ext_discount_amt], file_type=parquet, predicate=ss_quantity@0 >= 21 AND ss_quantity@0 <= 40, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 21 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 40, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_net_paid)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 21 AND ss_quantity@0 <= 40, projection=[ss_net_paid@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_net_paid], file_type=parquet, predicate=ss_quantity@0 >= 21 AND ss_quantity@0 <= 40, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 21 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 40, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 41 AND ss_quantity@0 <= 60 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity], file_type=parquet, predicate=ss_quantity@0 >= 41 AND ss_quantity@0 <= 60, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 41 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 60, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 41 AND ss_quantity@0 <= 60, projection=[ss_ext_discount_amt@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_ext_discount_amt], file_type=parquet, predicate=ss_quantity@0 >= 41 AND ss_quantity@0 <= 60, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 41 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 60, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_net_paid)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 41 AND ss_quantity@0 <= 60, projection=[ss_net_paid@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_net_paid], file_type=parquet, predicate=ss_quantity@0 >= 41 AND ss_quantity@0 <= 60, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 41 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 60, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 61 AND ss_quantity@0 <= 80 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity], file_type=parquet, predicate=ss_quantity@0 >= 61 AND ss_quantity@0 <= 80, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 61 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 80, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 61 AND ss_quantity@0 <= 80, projection=[ss_ext_discount_amt@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_ext_discount_amt], file_type=parquet, predicate=ss_quantity@0 >= 61 AND ss_quantity@0 <= 80, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 61 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 80, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_net_paid)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 61 AND ss_quantity@0 <= 80, projection=[ss_net_paid@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_net_paid], file_type=parquet, predicate=ss_quantity@0 >= 61 AND ss_quantity@0 <= 80, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 61 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 80, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 81 AND ss_quantity@0 <= 100 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity], file_type=parquet, predicate=ss_quantity@0 >= 81 AND ss_quantity@0 <= 100, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 81 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 100, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 81 AND ss_quantity@0 <= 100, projection=[ss_ext_discount_amt@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_ext_discount_amt], file_type=parquet, predicate=ss_quantity@0 >= 81 AND ss_quantity@0 <= 100, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 81 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 100, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_net_paid)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 81 AND ss_quantity@0 <= 100, projection=[ss_net_paid@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_net_paid], file_type=parquet, predicate=ss_quantity@0 >= 81 AND ss_quantity@0 <= 100, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 81 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 100, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_10() -> Result<()> { + let display = test_tpcds_query(10).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [cd_gender@0 ASC NULLS LAST, cd_marital_status@1 ASC NULLS LAST, cd_education_status@2 ASC NULLS LAST, cd_purchase_estimate@4 ASC NULLS LAST, cd_credit_rating@6 ASC NULLS LAST, cd_dep_count@8 ASC NULLS LAST, cd_dep_employed_count@10 ASC NULLS LAST, cd_dep_college_count@12 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[cd_gender@0 ASC NULLS LAST, cd_marital_status@1 ASC NULLS LAST, cd_education_status@2 ASC NULLS LAST, cd_purchase_estimate@4 ASC NULLS LAST, cd_credit_rating@6 ASC NULLS LAST, cd_dep_count@8 ASC NULLS LAST, cd_dep_employed_count@10 ASC NULLS LAST, cd_dep_college_count@12 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[cd_gender@0 as cd_gender, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, count(Int64(1))@8 as cnt1, cd_purchase_estimate@3 as cd_purchase_estimate, count(Int64(1))@8 as cnt2, cd_credit_rating@4 as cd_credit_rating, count(Int64(1))@8 as cnt3, cd_dep_count@5 as cd_dep_count, count(Int64(1))@8 as cnt4, cd_dep_employed_count@6 as cd_dep_employed_count, count(Int64(1))@8 as cnt5, cd_dep_college_count@7 as cd_dep_college_count, count(Int64(1))@8 as cnt6] + │ AggregateExec: mode=FinalPartitioned, gby=[cd_gender@0 as cd_gender, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, cd_purchase_estimate@3 as cd_purchase_estimate, cd_credit_rating@4 as cd_credit_rating, cd_dep_count@5 as cd_dep_count, cd_dep_employed_count@6 as cd_dep_employed_count, cd_dep_college_count@7 as cd_dep_college_count], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cd_gender@0, cd_marital_status@1, cd_education_status@2, cd_purchase_estimate@3, cd_credit_rating@4, cd_dep_count@5, cd_dep_employed_count@6, cd_dep_college_count@7], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cd_gender@0 as cd_gender, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, cd_purchase_estimate@3 as cd_purchase_estimate, cd_credit_rating@4 as cd_credit_rating, cd_dep_count@5 as cd_dep_count, cd_dep_employed_count@6 as cd_dep_employed_count, cd_dep_college_count@7 as cd_dep_college_count], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: mark@8 OR mark@9, projection=[cd_gender@0, cd_marital_status@1, cd_education_status@2, cd_purchase_estimate@3, cd_credit_rating@4, cd_dep_count@5, cd_dep_employed_count@6, cd_dep_college_count@7] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightMark, on=[(cs_ship_customer_sk@0, CAST(c.c_customer_sk AS Float64)@10)], projection=[cd_gender@1, cd_marital_status@2, cd_education_status@3, cd_purchase_estimate@4, cd_credit_rating@5, cd_dep_count@6, cd_dep_employed_count@7, cd_dep_college_count@8, mark@9, mark@11] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_ship_customer_sk@3] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_education_status@3 as cd_education_status, cd_purchase_estimate@4 as cd_purchase_estimate, cd_credit_rating@5 as cd_credit_rating, cd_dep_count@6 as cd_dep_count, cd_dep_employed_count@7 as cd_dep_employed_count, cd_dep_college_count@8 as cd_dep_college_count, mark@9 as mark, CAST(c_customer_sk@0 AS Float64) as CAST(c.c_customer_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightMark, on=[(ws_bill_customer_sk@0, CAST(c.c_customer_sk AS Float64)@9)], projection=[c_customer_sk@0, cd_gender@1, cd_marital_status@2, cd_education_status@3, cd_purchase_estimate@4, cd_credit_rating@5, cd_dep_count@6, cd_dep_employed_count@7, cd_dep_college_count@8, mark@10] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_bill_customer_sk@3] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_education_status@3 as cd_education_status, cd_purchase_estimate@4 as cd_purchase_estimate, cd_credit_rating@5 as cd_credit_rating, cd_dep_count@6 as cd_dep_count, cd_dep_employed_count@7 as cd_dep_employed_count, cd_dep_college_count@8 as cd_dep_college_count, CAST(c_customer_sk@0 AS Float64) as CAST(c.c_customer_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(ss_customer_sk@0, CAST(c.c_customer_sk AS Float64)@9)], projection=[c_customer_sk@0, cd_gender@1, cd_marital_status@2, cd_education_status@3, cd_purchase_estimate@4, cd_credit_rating@5, cd_dep_count@6, cd_dep_employed_count@7, cd_dep_college_count@8] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_customer_sk@3] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_education_status@3 as cd_education_status, cd_purchase_estimate@4 as cd_purchase_estimate, cd_credit_rating@5 as cd_credit_rating, cd_dep_count@6 as cd_dep_count, cd_dep_employed_count@7 as cd_dep_employed_count, cd_dep_college_count@8 as cd_dep_college_count, CAST(c_customer_sk@0 AS Float64) as CAST(c.c_customer_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@1, CAST(customer_demographics.cd_demo_sk AS Float64)@9)], projection=[c_customer_sk@0, cd_gender@3, cd_marital_status@4, cd_education_status@5, cd_purchase_estimate@6, cd_credit_rating@7, cd_dep_count@8, cd_dep_employed_count@9, cd_dep_college_count@10] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@2)], projection=[c_customer_sk@1, c_current_cdemo_sk@2] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_education_status@3 as cd_education_status, cd_purchase_estimate@4 as cd_purchase_estimate, cd_credit_rating@5 as cd_credit_rating, cd_dep_count@6 as cd_dep_count, cd_dep_employed_count@7 as cd_dep_employed_count, cd_dep_college_count@8 as cd_dep_college_count, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002 AND d_moy@2 >= 1 AND d_moy@2 <= 4, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2002 AND d_moy@2 >= 1 AND d_moy@2 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002 AND d_moy@2 >= 1 AND d_moy@2 <= 4, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2002 AND d_moy@2 >= 1 AND d_moy@2 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002 AND d_moy@2 >= 1 AND d_moy@2 <= 4, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2002 AND d_moy@2 >= 1 AND d_moy@2 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ca_county@1 IN ([Rush County, Toole County, Jefferson County, Dona Ana County, La Porte County]), projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=ca_county@1 IN ([Rush County, Toole County, Jefferson County, Dona Ana County, La Porte County]), pruning_predicate=ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Rush County AND Rush County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Toole County AND Toole County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Jefferson County AND Jefferson County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Dona Ana County AND Dona Ana County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= La Porte County AND La Porte County <= ca_county_max@1, required_guarantees=[ca_county in (Dona Ana County, Jefferson County, La Porte County, Rush County, Toole County)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_11() -> Result<()> { + let display = test_tpcds_query(11).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [customer_id@0 ASC, customer_first_name@1 ASC, customer_last_name@2 ASC, customer_preferred_cust_flag@3 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[customer_id@0 ASC, customer_first_name@1 ASC, customer_last_name@2 ASC, customer_preferred_cust_flag@3 ASC], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], filter=CASE WHEN year_total@2 > Some(0),18,2 THEN CAST(year_total@3 AS Float64) / CAST(year_total@2 AS Float64) ELSE 0 END > CASE WHEN year_total@0 > Some(0),18,2 THEN CAST(year_total@1 AS Float64) / CAST(year_total@0 AS Float64) ELSE 0 END, projection=[customer_id@2, customer_first_name@3, customer_last_name@4, customer_preferred_cust_flag@5] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[customer_id@1 as customer_id, year_total@2 as year_total, customer_id@3 as customer_id, customer_first_name@4 as customer_first_name, customer_last_name@5 as customer_last_name, customer_preferred_cust_flag@6 as customer_preferred_cust_flag, year_total@7 as year_total, year_total@0 as year_total] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], projection=[year_total@1, customer_id@2, year_total@3, customer_id@4, customer_first_name@5, customer_last_name@6, customer_preferred_cust_flag@7, year_total@8] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)@1 as year_total] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)@1 > Some(0),18,2 + │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)@8 as sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)], ordering_mode=PartiallySorted([7]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@9 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)], ordering_mode=PartiallySorted([7]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, ws_ext_discount_amt@8 as ws_ext_discount_amt, ws_ext_list_price@9 as ws_ext_list_price, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@7)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, c_preferred_cust_flag@6, c_birth_country@7, c_login@8, c_email_address@9, ws_ext_discount_amt@11, ws_ext_list_price@12] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ws_sold_date_sk@9, ws_ext_discount_amt@11, ws_ext_list_price@12] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)@1 as year_total] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)@1 > Some(0),18,2 + │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)@8 as sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)], ordering_mode=PartiallySorted([7]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@9 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)], ordering_mode=PartiallySorted([7]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, ss_ext_discount_amt@8 as ss_ext_discount_amt, ss_ext_list_price@9 as ss_ext_list_price, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@7)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, c_preferred_cust_flag@6, c_birth_country@7, c_login@8, c_email_address@9, ss_ext_discount_amt@11, ss_ext_list_price@12] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ss_sold_date_sk@9, ss_ext_discount_amt@11, ss_ext_list_price@12] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, c_first_name@1 as customer_first_name, c_last_name@2 as customer_last_name, c_preferred_cust_flag@3 as customer_preferred_cust_flag, sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)@8 as year_total] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)], ordering_mode=PartiallySorted([7]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@9 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)], ordering_mode=PartiallySorted([7]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, ss_ext_discount_amt@8 as ss_ext_discount_amt, ss_ext_list_price@9 as ss_ext_list_price, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@7)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, c_preferred_cust_flag@6, c_birth_country@7, c_login@8, c_email_address@9, ss_ext_discount_amt@11, ss_ext_list_price@12] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ss_sold_date_sk@9, ss_ext_discount_amt@11, ss_ext_list_price@12] + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)@8 as year_total] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)], ordering_mode=PartiallySorted([7]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@9 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)], ordering_mode=PartiallySorted([7]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, ws_ext_discount_amt@8 as ws_ext_discount_amt, ws_ext_list_price@9 as ws_ext_list_price, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@7)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, c_preferred_cust_flag@6, c_birth_country@7, c_login@8, c_email_address@9, ws_ext_discount_amt@11, ws_ext_list_price@12] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ws_sold_date_sk@9, ws_ext_discount_amt@11, ws_ext_list_price@12] + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@8], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@8], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@8], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@8], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_12() -> Result<()> { + let display = test_tpcds_query(12).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_category@2 ASC NULLS LAST, i_class@3 ASC NULLS LAST, i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, revenueratio@6 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_category@2 ASC NULLS LAST, i_class@3 ASC NULLS LAST, i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, revenueratio@6 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_category@2 as i_category, i_class@3 as i_class, i_current_price@4 as i_current_price, sum(web_sales.ws_ext_sales_price)@5 as itemrevenue, CAST(sum(web_sales.ws_ext_sales_price)@5 AS Float64) * 100 / CAST(sum(sum(web_sales.ws_ext_sales_price)) PARTITION BY [item.i_class] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@6 AS Float64) as revenueratio] + │ WindowAggExec: wdw=[sum(sum(web_sales.ws_ext_sales_price)) PARTITION BY [item.i_class] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(sum(web_sales.ws_ext_sales_price)) PARTITION BY [item.i_class] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Decimal128(27, 2), nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + │ SortExec: expr=[i_class@3 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_class@3], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_category@2 as i_category, i_class@3 as i_class, i_current_price@4 as i_current_price], aggr=[sum(web_sales.ws_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, i_category@2, i_class@3, i_current_price@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id, i_item_desc@2 as i_item_desc, i_category@5 as i_category, i_class@4 as i_class, i_current_price@3 as i_current_price], aggr=[sum(web_sales.ws_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_ext_sales_price@3, i_item_id@4, i_item_desc@5, i_current_price@6, i_class@7, i_category@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@5 as ws_sold_date_sk, ws_ext_sales_price@6 as ws_ext_sales_price, i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price, i_class@3 as i_class, i_category@4 as i_category] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_item_id@1, i_item_desc@2, i_current_price@3, i_class@4, i_category@5, ws_sold_date_sk@6, ws_ext_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 1999-02-22 AND d_date@1 <= 1999-03-24, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 1999-02-22 AND d_date@1 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_category@5 = Sports OR i_category@5 = Books OR i_category@5 = Home + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@5 = Sports OR i_category@5 = Books OR i_category@5 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_13() -> Result<()> { + let display = test_tpcds_query(13).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[avg(store_sales.ss_quantity)@0 as avg1, avg(store_sales.ss_ext_sales_price)@1 as avg2, avg(store_sales.ss_ext_wholesale_cost)@2 as avg3, sum(store_sales.ss_ext_wholesale_cost)@3 as sum(store_sales.ss_ext_wholesale_cost)] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_quantity), avg(store_sales.ss_ext_sales_price), avg(store_sales.ss_ext_wholesale_cost), sum(store_sales.ss_ext_wholesale_cost)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_quantity), avg(store_sales.ss_ext_sales_price), avg(store_sales.ss_ext_wholesale_cost), sum(store_sales.ss_ext_wholesale_cost)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_sold_date_sk@0, CAST(date_dim.d_date_sk AS Float64)@1)], projection=[ss_quantity@1, ss_ext_sales_price@2, ss_ext_wholesale_cost@3] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@2)], filter=(ca_state@1 = TX OR ca_state@1 = OH) AND ss_net_profit@0 >= Some(10000),6,2 AND ss_net_profit@0 <= Some(20000),6,2 OR (ca_state@1 = OR OR ca_state@1 = NM OR ca_state@1 = KY) AND ss_net_profit@0 >= Some(15000),6,2 AND ss_net_profit@0 <= Some(30000),6,2 OR (ca_state@1 = VA OR ca_state@1 = TX OR ca_state@1 = MS) AND ss_net_profit@0 >= Some(5000),6,2 AND ss_net_profit@0 <= Some(25000),6,2, projection=[ss_sold_date_sk@0, ss_quantity@2, ss_ext_sales_price@3, ss_ext_wholesale_cost@4] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_hdemo_sk@1, CAST(household_demographics.hd_demo_sk AS Float64)@2)], filter=cd_marital_status@1 = M AND cd_education_status@2 = Advanced Degree AND ss_sales_price@0 >= Some(10000),5,2 AND ss_sales_price@0 <= Some(15000),5,2 AND hd_dep_count@3 = 3 OR cd_marital_status@1 = S AND cd_education_status@2 = College AND ss_sales_price@0 >= Some(5000),5,2 AND ss_sales_price@0 <= Some(10000),5,2 AND hd_dep_count@3 = 1 OR cd_marital_status@1 = W AND cd_education_status@2 = 2 yr Degree AND ss_sales_price@0 >= Some(15000),5,2 AND ss_sales_price@0 <= Some(20000),5,2 AND hd_dep_count@3 = 1, projection=[ss_sold_date_sk@0, ss_addr_sk@2, ss_quantity@3, ss_ext_sales_price@5, ss_ext_wholesale_cost@6, ss_net_profit@7] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_cdemo_sk@1, CAST(customer_demographics.cd_demo_sk AS Float64)@3)], filter=cd_marital_status@1 = M AND cd_education_status@2 = Advanced Degree AND ss_sales_price@0 >= Some(10000),5,2 AND ss_sales_price@0 <= Some(15000),5,2 OR cd_marital_status@1 = S AND cd_education_status@2 = College AND ss_sales_price@0 >= Some(5000),5,2 AND ss_sales_price@0 <= Some(10000),5,2 OR cd_marital_status@1 = W AND cd_education_status@2 = 2 yr Degree AND ss_sales_price@0 >= Some(15000),5,2 AND ss_sales_price@0 <= Some(20000),5,2, projection=[ss_sold_date_sk@0, ss_hdemo_sk@2, ss_addr_sk@3, ss_quantity@4, ss_sales_price@5, ss_ext_sales_price@6, ss_ext_wholesale_cost@7, ss_net_profit@8, cd_marital_status@10, cd_education_status@11] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@4)], projection=[ss_sold_date_sk@2, ss_cdemo_sk@3, ss_hdemo_sk@4, ss_addr_sk@5, ss_quantity@7, ss_sales_price@8, ss_ext_sales_price@9, ss_ext_wholesale_cost@10, ss_net_profit@11] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: (ss_net_profit@9 >= Some(10000),6,2 AND ss_net_profit@9 <= Some(20000),6,2 OR ss_net_profit@9 >= Some(15000),6,2 AND ss_net_profit@9 <= Some(30000),6,2 OR ss_net_profit@9 >= Some(5000),6,2 AND ss_net_profit@9 <= Some(25000),6,2) AND (ss_sales_price@6 >= Some(10000),5,2 AND ss_sales_price@6 <= Some(15000),5,2 OR ss_sales_price@6 >= Some(5000),5,2 AND ss_sales_price@6 <= Some(10000),5,2 OR ss_sales_price@6 >= Some(15000),5,2 AND ss_sales_price@6 <= Some(20000),5,2) + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_quantity, ss_sales_price, ss_ext_sales_price, ss_ext_wholesale_cost, ss_net_profit], file_type=parquet, predicate=(ss_net_profit@9 >= Some(10000),6,2 AND ss_net_profit@9 <= Some(20000),6,2 OR ss_net_profit@9 >= Some(15000),6,2 AND ss_net_profit@9 <= Some(30000),6,2 OR ss_net_profit@9 >= Some(5000),6,2 AND ss_net_profit@9 <= Some(25000),6,2) AND (ss_sales_price@6 >= Some(10000),5,2 AND ss_sales_price@6 <= Some(15000),5,2 OR ss_sales_price@6 >= Some(5000),5,2 AND ss_sales_price@6 <= Some(10000),5,2 OR ss_sales_price@6 >= Some(15000),5,2 AND ss_sales_price@6 <= Some(20000),5,2) AND DynamicFilter [ empty ], pruning_predicate=(ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(10000),6,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(20000),6,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(15000),6,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(30000),6,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(5000),6,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(25000),6,2) AND (ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(10000),5,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(15000),5,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(5000),5,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(10000),5,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(15000),5,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(20000),5,2), required_guarantees=[] + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cd_marital_status@1 = M AND cd_education_status@2 = Advanced Degree OR cd_marital_status@1 = S AND cd_education_status@2 = College OR cd_marital_status@1 = W AND cd_education_status@2 = 2 yr Degree + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_marital_status@1 = M AND cd_education_status@2 = Advanced Degree OR cd_marital_status@1 = S AND cd_education_status@2 = College OR cd_marital_status@1 = W AND cd_education_status@2 = 2 yr Degree, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, Advanced Degree, College), cd_marital_status in (M, S, W)] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, hd_dep_count@1 as hd_dep_count, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: (__common_expr_3@0 OR hd_dep_count@2 = 1) AND __common_expr_3@0, projection=[hd_demo_sk@1, hd_dep_count@2] + │ ProjectionExec: expr=[hd_dep_count@1 = 3 OR hd_dep_count@1 = 1 as __common_expr_3, hd_demo_sk@0 as hd_demo_sk, hd_dep_count@1 as hd_dep_count] + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count], file_type=parquet + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_state@1 as ca_state, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: __common_expr_4@0 AND __common_expr_4@0 AND ca_country@3 = United States AND ca_state@2 IN ([TX, OH, OR, NM, KY, VA, MS]), projection=[ca_address_sk@1, ca_state@2] + │ ProjectionExec: expr=[__common_expr_5@0 OR ca_state@2 = OH OR ca_state@2 = OR OR ca_state@2 = NM OR ca_state@2 = KY OR ca_state@2 = VA OR __common_expr_5@0 OR ca_state@2 = MS as __common_expr_4, ca_address_sk@1 as ca_address_sk, ca_state@2 as ca_state, ca_country@3 as ca_country] + │ ProjectionExec: expr=[ca_state@1 = TX as __common_expr_5, ca_address_sk@0 as ca_address_sk, ca_state@1 as ca_state, ca_country@2 as ca_country] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@2 = United States AND ca_state@1 IN ([TX, OH, OR, NM, KY, VA, MS]), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NM AND NM <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (KY, MS, NM, OH, OR, TX, VA)] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_14() -> Result<()> { + let display = test_tpcds_query(14).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [channel@0 ASC, i_brand_id@1 ASC, i_class_id@2 ASC, i_category_id@3 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, i_brand_id@1 ASC, i_class_id@2 ASC, i_category_id@3 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[channel@0 as channel, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, sum(y.sales)@5 as sum_sales, sum(y.number_sales)@6 as sum_number_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, __grouping_id@4 as __grouping_id], aggr=[sum(y.sales), sum(y.number_sales)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([channel@0, i_brand_id@1, i_class_id@2, i_category_id@3, __grouping_id@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[(NULL as channel, NULL as i_brand_id, NULL as i_class_id, NULL as i_category_id), (channel@0 as channel, NULL as i_brand_id, NULL as i_class_id, NULL as i_category_id), (channel@0 as channel, i_brand_id@1 as i_brand_id, NULL as i_class_id, NULL as i_category_id), (channel@0 as channel, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, NULL as i_category_id), (channel@0 as channel, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id)], aggr=[sum(y.sales), sum(y.number_sales)] + │ InterleaveExec + │ ProjectionExec: expr=[store as channel, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, sum(store_sales.ss_quantity * store_sales.ss_list_price)@3 as sales, count(Int64(1))@4 as number_sales] + │ ProjectionExec: expr=[i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, sum(store_sales.ss_quantity * store_sales.ss_list_price)@4 as sum(store_sales.ss_quantity * store_sales.ss_list_price), count(Int64(1))@5 as count(Int64(1)), average_sales@0 as average_sales] + │ NestedLoopJoinExec: join_type=Inner, filter=sum(store_sales.ss_quantity * store_sales.ss_list_price)@0 > average_sales@1 + │ ProjectionExec: expr=[avg(sq2.quantity * sq2.list_price)@0 as average_sales] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] + │ UnionExec + │ ProjectionExec: expr=[ss_quantity@0 as quantity, ss_list_price@1 as list_price] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_quantity@3, ss_list_price@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[cs_quantity@0 as quantity, cs_list_price@1 as list_price] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_quantity@3, cs_list_price@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ws_quantity@0 as quantity, ws_list_price@1 as list_price] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_quantity@3, ws_list_price@4] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ AggregateExec: mode=FinalPartitioned, gby=[i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id], aggr=[sum(store_sales.ss_quantity * store_sales.ss_list_price), count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id], aggr=[sum(store_sales.ss_quantity * store_sales.ss_list_price), count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(ss_item_sk@0, ss_item_sk@0)], projection=[ss_quantity@1, ss_list_price@2, i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_quantity@4, ss_list_price@5, i_brand_id@6, i_class_id@7, i_category_id@8] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@3 as ss_sold_date_sk, ss_item_sk@4 as ss_item_sk, ss_quantity@5 as ss_quantity, ss_list_price@6 as ss_list_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ss_sold_date_sk@4, ss_item_sk@5, ss_quantity@6, ss_list_price@7] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[i_item_sk@0 as ss_item_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(brand_id@0, i_brand_id@1), (class_id@1, i_class_id@2), (category_id@2, i_category_id@3)], projection=[i_item_sk@3] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d3.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@3 as ws_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ws_sold_date_sk@4] + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ AggregateExec: mode=SinglePartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d2.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@3 as cs_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, cs_sold_date_sk@4] + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ AggregateExec: mode=FinalPartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ ProjectionExec: expr=[i_brand_id@0 as brand_id, i_class_id@1 as class_id, i_category_id@2 as category_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@3 as ss_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ss_sold_date_sk@4] + │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[catalog as channel, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)@3 as sales, count(Int64(1))@4 as number_sales] + │ ProjectionExec: expr=[i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)@4 as sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price), count(Int64(1))@5 as count(Int64(1)), average_sales@0 as average_sales] + │ NestedLoopJoinExec: join_type=Inner, filter=sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)@0 > average_sales@1 + │ ProjectionExec: expr=[avg(sq2.quantity * sq2.list_price)@0 as average_sales] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] + │ UnionExec + │ ProjectionExec: expr=[ss_quantity@0 as quantity, ss_list_price@1 as list_price] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_quantity@3, ss_list_price@4] + │ CoalescePartitionsExec + │ [Stage 16] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[cs_quantity@0 as quantity, cs_list_price@1 as list_price] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_quantity@3, cs_list_price@4] + │ CoalescePartitionsExec + │ [Stage 17] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ws_quantity@0 as quantity, ws_list_price@1 as list_price] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_quantity@3, ws_list_price@4] + │ CoalescePartitionsExec + │ [Stage 18] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ AggregateExec: mode=FinalPartitioned, gby=[i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id], aggr=[sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price), count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id], aggr=[sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price), count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(cs_item_sk@0, ss_item_sk@0)], projection=[cs_quantity@1, cs_list_price@2, i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_item_sk@3, cs_quantity@4, cs_list_price@5, i_brand_id@6, i_class_id@7, i_category_id@8] + │ CoalescePartitionsExec + │ [Stage 19] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@3 as cs_sold_date_sk, cs_item_sk@4 as cs_item_sk, cs_quantity@5 as cs_quantity, cs_list_price@6 as cs_list_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, cs_sold_date_sk@4, cs_item_sk@5, cs_quantity@6, cs_list_price@7] + │ [Stage 20] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 21] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[i_item_sk@0 as ss_item_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(brand_id@0, i_brand_id@1), (class_id@1, i_class_id@2), (category_id@2, i_category_id@3)], projection=[i_item_sk@3] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d3.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 22] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@3 as ws_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ws_sold_date_sk@4] + │ [Stage 23] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 24] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ AggregateExec: mode=SinglePartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d2.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 25] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@3 as cs_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, cs_sold_date_sk@4] + │ [Stage 26] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 27] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ AggregateExec: mode=FinalPartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ ProjectionExec: expr=[i_brand_id@0 as brand_id, i_class_id@1 as class_id, i_category_id@2 as category_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 28] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@3 as ss_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ss_sold_date_sk@4] + │ [Stage 29] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 30] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[web as channel, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, sum(web_sales.ws_quantity * web_sales.ws_list_price)@3 as sales, count(Int64(1))@4 as number_sales] + │ ProjectionExec: expr=[i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, sum(web_sales.ws_quantity * web_sales.ws_list_price)@4 as sum(web_sales.ws_quantity * web_sales.ws_list_price), count(Int64(1))@5 as count(Int64(1)), average_sales@0 as average_sales] + │ NestedLoopJoinExec: join_type=Inner, filter=sum(web_sales.ws_quantity * web_sales.ws_list_price)@0 > average_sales@1 + │ ProjectionExec: expr=[avg(sq2.quantity * sq2.list_price)@0 as average_sales] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] + │ UnionExec + │ ProjectionExec: expr=[ss_quantity@0 as quantity, ss_list_price@1 as list_price] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_quantity@3, ss_list_price@4] + │ CoalescePartitionsExec + │ [Stage 31] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[cs_quantity@0 as quantity, cs_list_price@1 as list_price] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_quantity@3, cs_list_price@4] + │ CoalescePartitionsExec + │ [Stage 32] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ws_quantity@0 as quantity, ws_list_price@1 as list_price] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_quantity@3, ws_list_price@4] + │ CoalescePartitionsExec + │ [Stage 33] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ AggregateExec: mode=FinalPartitioned, gby=[i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id], aggr=[sum(web_sales.ws_quantity * web_sales.ws_list_price), count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id], aggr=[sum(web_sales.ws_quantity * web_sales.ws_list_price), count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(ws_item_sk@0, ss_item_sk@0)], projection=[ws_quantity@1, ws_list_price@2, i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_quantity@4, ws_list_price@5, i_brand_id@6, i_class_id@7, i_category_id@8] + │ CoalescePartitionsExec + │ [Stage 34] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@3 as ws_sold_date_sk, ws_item_sk@4 as ws_item_sk, ws_quantity@5 as ws_quantity, ws_list_price@6 as ws_list_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ws_sold_date_sk@4, ws_item_sk@5, ws_quantity@6, ws_list_price@7] + │ [Stage 35] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 36] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[i_item_sk@0 as ss_item_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(brand_id@0, i_brand_id@1), (class_id@1, i_class_id@2), (category_id@2, i_category_id@3)], projection=[i_item_sk@3] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d3.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 37] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@3 as ws_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ws_sold_date_sk@4] + │ [Stage 38] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 39] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ AggregateExec: mode=SinglePartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d2.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 40] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@3 as cs_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, cs_sold_date_sk@4] + │ [Stage 41] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 42] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ AggregateExec: mode=FinalPartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ ProjectionExec: expr=[i_brand_id@0 as brand_id, i_class_id@1 as class_id, i_category_id@2 as category_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 43] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@3 as ss_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ss_sold_date_sk@4] + │ [Stage 44] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 45] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 11, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 17 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 18 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 19 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 11, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 20 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 21 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 22 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 23 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 24 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 25 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 26 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 27 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 28 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 29 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 30 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 31 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 32 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 33 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 34 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 11, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 35 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 36 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 37 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 38 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 39 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 40 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 41 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 42 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 43 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 44 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 45 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_15() -> Result<()> { + let display = test_tpcds_query(15).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [ca_zip@0 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[ca_zip@0 ASC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[ca_zip@0 as ca_zip], aggr=[sum(catalog_sales.cs_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ca_zip@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_zip@1 as ca_zip], aggr=[sum(catalog_sales.cs_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_sales_price@3, ca_zip@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_sales_price@2 as cs_sales_price, ca_zip@0 as ca_zip] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@2)], filter=substr(ca_zip@2, 1, 5) IN ([85669, 86197, 88274, 83405, 86475, 85392, 85460, 80348, 81792]) OR ca_state@1 = CA OR ca_state@1 = WA OR ca_state@1 = GA OR cs_sales_price@0 > Some(50000),5,2, projection=[ca_zip@2, cs_sold_date_sk@3, cs_sales_price@4] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_qoy@2 = 2 AND d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@2 = 2 AND d_year@1 = 2001, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ca_address_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state, ca_zip], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_current_addr_sk@2], 3), input_partitions=3 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_sales_price@2 as cs_sales_price, c_current_addr_sk@0 as c_current_addr_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@2, cs_bill_customer_sk@1)], projection=[c_current_addr_sk@1, cs_sold_date_sk@3, cs_sales_price@5] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@2], 6), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_16() -> Result<()> { + let display = test_tpcds_query(16).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(alias1)@0 as order count, sum(alias2)@1 as total shipping cost, sum(alias3)@2 as total net profit] + │ GlobalLimitExec: skip=0, fetch=100 + │ AggregateExec: mode=Final, gby=[], aggr=[count(alias1), sum(alias2), sum(alias3)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[count(alias1), sum(alias2), sum(alias3)] + │ AggregateExec: mode=FinalPartitioned, gby=[alias1@0 as alias1], aggr=[alias2, alias3] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([alias1@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cs_order_number@0 as alias1], aggr=[alias2, alias3] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(cs_order_number@0, cr_order_number@0)] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(cs_order_number@1, cs_order_number@1)], filter=cs_warehouse_sk@0 != cs_warehouse_sk@1, projection=[cs_order_number@1, cs_ext_ship_cost@2, cs_net_profit@3] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(call_center.cc_call_center_sk AS Float64)@1, cs_call_center_sk@0)], projection=[cs_warehouse_sk@3, cs_order_number@4, cs_ext_ship_cost@5, cs_net_profit@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer_address.ca_address_sk AS Float64)@1, cs_ship_addr_sk@0)], projection=[cs_call_center_sk@3, cs_warehouse_sk@4, cs_order_number@5, cs_ext_ship_cost@6, cs_net_profit@7] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_ship_date_sk@0)], projection=[cs_ship_addr_sk@3, cs_call_center_sk@4, cs_warehouse_sk@5, cs_order_number@6, cs_ext_ship_cost@7, cs_net_profit@8] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_ship_date_sk, cs_ship_addr_sk, cs_call_center_sk, cs_warehouse_sk, cs_order_number, cs_ext_ship_cost, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_warehouse_sk, cs_order_number], file_type=parquet + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_order_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[cc_call_center_sk@0 as cc_call_center_sk, CAST(cc_call_center_sk@0 AS Float64) as CAST(call_center.cc_call_center_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cc_county@1 = Williamson County, projection=[cc_call_center_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/call_center/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-3.parquet]]}, projection=[cc_call_center_sk, cc_county], file_type=parquet, predicate=cc_county@1 = Williamson County, pruning_predicate=cc_county_null_count@2 != row_count@3 AND cc_county_min@0 <= Williamson County AND Williamson County <= cc_county_max@1, required_guarantees=[cc_county in (Williamson County)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ca_state@1 = GA, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@1 = GA, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1, required_guarantees=[ca_state in (GA)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2002-02-01 AND d_date@1 <= 2002-04-02, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2002-02-01 AND d_date@1 <= 2002-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2002-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2002-04-02, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_17() -> Result<()> { + let display = test_tpcds_query(17).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_item_id@0 ASC, i_item_desc@1 ASC, s_state@2 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC, i_item_desc@1 ASC, s_state@2 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, s_state@2 as s_state, count(store_sales.ss_quantity)@3 as store_sales_quantitycount, avg(store_sales.ss_quantity)@4 as store_sales_quantityave, stddev(store_sales.ss_quantity)@5 as store_sales_quantitystdev, stddev(store_sales.ss_quantity)@5 / avg(store_sales.ss_quantity)@4 as store_sales_quantitycov, count(store_returns.sr_return_quantity)@6 as store_returns_quantitycount, avg(store_returns.sr_return_quantity)@7 as store_returns_quantityave, stddev(store_returns.sr_return_quantity)@8 as store_returns_quantitystdev, stddev(store_returns.sr_return_quantity)@8 / avg(store_returns.sr_return_quantity)@7 as store_returns_quantitycov, count(catalog_sales.cs_quantity)@9 as catalog_sales_quantitycount, avg(catalog_sales.cs_quantity)@10 as catalog_sales_quantityave, stddev(catalog_sales.cs_quantity)@11 as catalog_sales_quantitystdev, stddev(catalog_sales.cs_quantity)@11 / avg(catalog_sales.cs_quantity)@10 as catalog_sales_quantitycov] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, s_state@2 as s_state], aggr=[count(store_sales.ss_quantity), avg(store_sales.ss_quantity), stddev(store_sales.ss_quantity), count(store_returns.sr_return_quantity), avg(store_returns.sr_return_quantity), stddev(store_returns.sr_return_quantity), count(catalog_sales.cs_quantity), avg(catalog_sales.cs_quantity), stddev(catalog_sales.cs_quantity)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, s_state@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@4 as i_item_id, i_item_desc@5 as i_item_desc, s_state@3 as s_state], aggr=[count(store_sales.ss_quantity), avg(store_sales.ss_quantity), stddev(store_sales.ss_quantity), count(store_returns.sr_return_quantity), avg(store_returns.sr_return_quantity), stddev(store_returns.sr_return_quantity), count(catalog_sales.cs_quantity), avg(catalog_sales.cs_quantity), stddev(catalog_sales.cs_quantity)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_quantity@1, sr_return_quantity@2, cs_quantity@3, s_state@4, i_item_id@6, i_item_desc@7] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_quantity@2 as ss_quantity, sr_return_quantity@3 as sr_return_quantity, cs_quantity@4 as cs_quantity, s_state@0 as s_state] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, ss_store_sk@1)], projection=[s_state@1, ss_item_sk@3, ss_quantity@5, sr_return_quantity@6, cs_quantity@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d3.d_date_sk AS Float64)@1, cs_sold_date_sk@4)], projection=[ss_item_sk@2, ss_store_sk@3, ss_quantity@4, sr_return_quantity@5, cs_quantity@7] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d2.d_date_sk AS Float64)@1, sr_returned_date_sk@3)], projection=[ss_item_sk@2, ss_store_sk@3, ss_quantity@4, sr_return_quantity@6, cs_sold_date_sk@7, cs_quantity@8] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_quantity@5, sr_returned_date_sk@6, sr_return_quantity@7, cs_sold_date_sk@8, cs_quantity@9] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@2 as ss_sold_date_sk, ss_item_sk@3 as ss_item_sk, ss_store_sk@4 as ss_store_sk, ss_quantity@5 as ss_quantity, sr_returned_date_sk@6 as sr_returned_date_sk, sr_return_quantity@7 as sr_return_quantity, cs_sold_date_sk@0 as cs_sold_date_sk, cs_quantity@1 as cs_quantity] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cs_bill_customer_sk@1, sr_customer_sk@6), (cs_item_sk@2, sr_item_sk@5)], projection=[cs_sold_date_sk@0, cs_quantity@3, ss_sold_date_sk@4, ss_item_sk@5, ss_store_sk@6, ss_quantity@7, sr_returned_date_sk@8, sr_return_quantity@11] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_state@1 as s_state, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_quarter_name@1 = 2001Q1 OR d_quarter_name@1 = 2001Q2 OR d_quarter_name@1 = 2001Q3, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@1 = 2001Q1 OR d_quarter_name@1 = 2001Q2 OR d_quarter_name@1 = 2001Q3, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q2 AND 2001Q2 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q3 AND 2001Q3 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1, 2001Q2, 2001Q3)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_quarter_name@1 = 2001Q1 OR d_quarter_name@1 = 2001Q2 OR d_quarter_name@1 = 2001Q3, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@1 = 2001Q1 OR d_quarter_name@1 = 2001Q2 OR d_quarter_name@1 = 2001Q3, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q2 AND 2001Q2 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q3 AND 2001Q3 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1, 2001Q2, 2001Q3)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_quarter_name@1 = 2001Q1, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@1 = 2001Q1, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1, cs_item_sk@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_customer_sk@6, sr_item_sk@5], 3), input_partitions=3 + │ ProjectionExec: expr=[ss_sold_date_sk@4 as ss_sold_date_sk, ss_item_sk@5 as ss_item_sk, ss_store_sk@6 as ss_store_sk, ss_quantity@7 as ss_quantity, sr_returned_date_sk@0 as sr_returned_date_sk, sr_item_sk@1 as sr_item_sk, sr_customer_sk@2 as sr_customer_sk, sr_return_quantity@3 as sr_return_quantity] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_customer_sk@2, ss_customer_sk@2), (sr_item_sk@1, ss_item_sk@1), (sr_ticket_number@3, ss_ticket_number@4)], projection=[sr_returned_date_sk@0, sr_item_sk@1, sr_customer_sk@2, sr_return_quantity@4, ss_sold_date_sk@5, ss_item_sk@6, ss_store_sk@8, ss_quantity@10] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_customer_sk@2, sr_item_sk@1, sr_ticket_number@3], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_customer_sk@2, ss_item_sk@1, ss_ticket_number@4], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_18() -> Result<()> { + let display = test_tpcds_query(18).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [ca_country@1 ASC, ca_state@2 ASC, ca_county@3 ASC, i_item_id@0 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[ca_country@1 ASC, ca_state@2 ASC, ca_county@3 ASC, i_item_id@0 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, ca_country@1 as ca_country, ca_state@2 as ca_state, ca_county@3 as ca_county, avg(catalog_sales.cs_quantity)@5 as agg1, avg(catalog_sales.cs_list_price)@6 as agg2, avg(catalog_sales.cs_coupon_amt)@7 as agg3, avg(catalog_sales.cs_sales_price)@8 as agg4, avg(catalog_sales.cs_net_profit)@9 as agg5, avg(customer.c_birth_year)@10 as agg6, avg(cd1.cd_dep_count)@11 as agg7] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, ca_country@1 as ca_country, ca_state@2 as ca_state, ca_county@3 as ca_county, __grouping_id@4 as __grouping_id], aggr=[avg(catalog_sales.cs_quantity), avg(catalog_sales.cs_list_price), avg(catalog_sales.cs_coupon_amt), avg(catalog_sales.cs_sales_price), avg(catalog_sales.cs_net_profit), avg(customer.c_birth_year), avg(cd1.cd_dep_count)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0, ca_country@1, ca_state@2, ca_county@3, __grouping_id@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[(NULL as i_item_id, NULL as ca_country, NULL as ca_state, NULL as ca_county), (i_item_id@10 as i_item_id, NULL as ca_country, NULL as ca_state, NULL as ca_county), (i_item_id@10 as i_item_id, ca_country@9 as ca_country, NULL as ca_state, NULL as ca_county), (i_item_id@10 as i_item_id, ca_country@9 as ca_country, ca_state@8 as ca_state, NULL as ca_county), (i_item_id@10 as i_item_id, ca_country@9 as ca_country, ca_state@8 as ca_state, ca_county@7 as ca_county)], aggr=[avg(catalog_sales.cs_quantity), avg(catalog_sales.cs_list_price), avg(catalog_sales.cs_coupon_amt), avg(catalog_sales.cs_sales_price), avg(catalog_sales.cs_net_profit), avg(customer.c_birth_year), avg(cd1.cd_dep_count)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, i_item_sk@0)], projection=[cs_quantity@1, cs_list_price@2, cs_sales_price@3, cs_coupon_amt@4, cs_net_profit@5, cd_dep_count@6, c_birth_year@7, ca_county@8, ca_state@9, ca_country@10, i_item_id@12] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_item_sk@3, cs_quantity@4, cs_list_price@5, cs_sales_price@6, cs_coupon_amt@7, cs_net_profit@8, cd_dep_count@9, c_birth_year@10, ca_county@11, ca_state@12, ca_country@13] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@3 as cs_sold_date_sk, cs_item_sk@4 as cs_item_sk, cs_quantity@5 as cs_quantity, cs_list_price@6 as cs_list_price, cs_sales_price@7 as cs_sales_price, cs_coupon_amt@8 as cs_coupon_amt, cs_net_profit@9 as cs_net_profit, cd_dep_count@10 as cd_dep_count, c_birth_year@11 as c_birth_year, ca_county@0 as ca_county, ca_state@1 as ca_state, ca_country@2 as ca_country] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@8)], projection=[ca_county@1, ca_state@2, ca_country@3, cs_sold_date_sk@4, cs_item_sk@5, cs_quantity@6, cs_list_price@7, cs_sales_price@8, cs_coupon_amt@9, cs_net_profit@10, cd_dep_count@11, c_birth_year@13] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c_current_cdemo_sk@8, CAST(cd2.cd_demo_sk AS Float64)@1)], projection=[cs_sold_date_sk@0, cs_item_sk@1, cs_quantity@2, cs_list_price@3, cs_sales_price@4, cs_coupon_amt@5, cs_net_profit@6, cd_dep_count@7, c_current_addr_sk@9, c_birth_year@10] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_current_cdemo_sk@8], 3), input_partitions=3 + │ ProjectionExec: expr=[cs_sold_date_sk@3 as cs_sold_date_sk, cs_item_sk@4 as cs_item_sk, cs_quantity@5 as cs_quantity, cs_list_price@6 as cs_list_price, cs_sales_price@7 as cs_sales_price, cs_coupon_amt@8 as cs_coupon_amt, cs_net_profit@9 as cs_net_profit, cd_dep_count@10 as cd_dep_count, c_current_cdemo_sk@0 as c_current_cdemo_sk, c_current_addr_sk@1 as c_current_addr_sk, c_birth_year@2 as c_birth_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@4, cs_bill_customer_sk@1)], projection=[c_current_cdemo_sk@1, c_current_addr_sk@2, c_birth_year@3, cs_sold_date_sk@5, cs_item_sk@7, cs_quantity@8, cs_list_price@9, cs_sales_price@10, cs_coupon_amt@11, cs_net_profit@12, cd_dep_count@13] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_bill_customer_sk@2 as cs_bill_customer_sk, cs_item_sk@3 as cs_item_sk, cs_quantity@4 as cs_quantity, cs_list_price@5 as cs_list_price, cs_sales_price@6 as cs_sales_price, cs_coupon_amt@7 as cs_coupon_amt, cs_net_profit@8 as cs_net_profit, cd_dep_count@0 as cd_dep_count] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(cd1.cd_demo_sk AS Float64)@2, cs_bill_cdemo_sk@2)], projection=[cd_dep_count@1, cs_sold_date_sk@3, cs_bill_customer_sk@4, cs_item_sk@6, cs_quantity@7, cs_list_price@8, cs_sales_price@9, cs_coupon_amt@10, cs_net_profit@11] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1998, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 1998, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1, required_guarantees=[d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ca_state@2 IN ([MS, IN, ND, OK, NM, VA, MS]) + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_county, ca_state, ca_country], file_type=parquet, predicate=ca_state@2 IN ([MS, IN, ND, OK, NM, VA, MS]), pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IN AND IN <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= ND AND ND <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OK AND OK <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NM AND NM <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= VA AND VA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1, required_guarantees=[ca_state in (IN, MS, ND, NM, OK, VA)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_cdemo_sk@1 as c_current_cdemo_sk, c_current_addr_sk@2 as c_current_addr_sk, c_birth_year@3 as c_birth_year, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: c_birth_month@3 IN (SET) ([1, 6, 8, 9, 12, 2]), projection=[c_customer_sk@0, c_current_cdemo_sk@1, c_current_addr_sk@2, c_birth_year@4] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk, c_birth_month, c_birth_year], file_type=parquet, predicate=c_birth_month@3 IN (SET) ([1, 6, 8, 9, 12, 2]) AND DynamicFilter [ empty ], pruning_predicate=c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 1 AND 1 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 6 AND 6 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 8 AND 8 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 9 AND 9 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 12 AND 12 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 2 AND 2 <= c_birth_month_max@1, required_guarantees=[c_birth_month in (1, 12, 2, 6, 8, 9)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(cd1.cd_demo_sk AS Float64)@2], 3), input_partitions=3 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, cd_dep_count@1 as cd_dep_count, CAST(cd_demo_sk@0 AS Float64) as CAST(cd1.cd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cd_gender@1 = F AND cd_education_status@2 = Unknown, projection=[cd_demo_sk@0, cd_dep_count@3] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_education_status, cd_dep_count], file_type=parquet, predicate=cd_gender@1 = F AND cd_education_status@2 = Unknown, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= F AND F <= cd_gender_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Unknown AND Unknown <= cd_education_status_max@5, required_guarantees=[cd_education_status in (Unknown), cd_gender in (F)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_bill_cdemo_sk@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_bill_cdemo_sk, cs_item_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(cd2.cd_demo_sk AS Float64)@1], 3), input_partitions=3 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(cd2.cd_demo_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_19() -> Result<()> { + let display = test_tpcds_query(19).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[brand_id@0 as brand_id, brand@1 as brand, i_manufact_id@2 as i_manufact_id, i_manufact@3 as i_manufact, ext_price@4 as ext_price] + │ SortPreservingMergeExec: [ext_price@4 DESC, i_brand@5 ASC NULLS LAST, i_brand_id@6 ASC NULLS LAST, i_manufact_id@2 ASC NULLS LAST, i_manufact@3 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[ext_price@4 DESC, brand@1 ASC NULLS LAST, brand_id@0 ASC NULLS LAST, i_manufact_id@2 ASC NULLS LAST, i_manufact@3 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_brand_id@1 as brand_id, i_brand@0 as brand, i_manufact_id@2 as i_manufact_id, i_manufact@3 as i_manufact, sum(store_sales.ss_ext_sales_price)@4 as ext_price, i_brand@0 as i_brand, i_brand_id@1 as i_brand_id] + │ AggregateExec: mode=FinalPartitioned, gby=[i_brand@0 as i_brand, i_brand_id@1 as i_brand_id, i_manufact_id@2 as i_manufact_id, i_manufact@3 as i_manufact], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_brand@0, i_brand_id@1, i_manufact_id@2, i_manufact@3], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_brand@2 as i_brand, i_brand_id@1 as i_brand_id, i_manufact_id@3 as i_manufact_id, i_manufact@4 as i_manufact], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, ss_store_sk@0)], filter=substr(ca_zip@0, 1, 5) != substr(s_zip@1, 1, 5), projection=[ss_ext_sales_price@4, i_brand_id@5, i_brand@6, i_manufact_id@7, i_manufact@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c_current_addr_sk@6, ca_address_sk@0)], projection=[ss_store_sk@0, ss_ext_sales_price@1, i_brand_id@2, i_brand@3, i_manufact_id@4, i_manufact@5, ca_zip@8] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_zip@1 as s_zip, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_zip], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_current_addr_sk@6], 3), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ss_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@2)], projection=[ss_store_sk@1, ss_ext_sales_price@2, i_brand_id@3, i_brand@4, i_manufact_id@5, i_manufact@6, c_current_addr_sk@8] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=1 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_customer_sk@0], 6), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_customer_sk@1, ss_store_sk@2, ss_ext_sales_price@3, i_brand_id@5, i_brand@6, i_manufact_id@7, i_manufact@8] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_customer_sk@4, ss_store_sk@5, ss_ext_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_manager_id@5 = 8, projection=[i_item_sk@0, i_brand_id@1, i_brand@2, i_manufact_id@3, i_manufact@4] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manufact_id, i_manufact, i_manager_id], file_type=parquet, predicate=i_manager_id@5 = 8 AND DynamicFilter [ empty ], pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 8 AND 8 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (8)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1998, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 11 AND d_year@1 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@2], 6), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ca_address_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_20() -> Result<()> { + let display = test_tpcds_query(20).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_category@2 ASC, i_class@3 ASC, i_item_id@0 ASC, i_item_desc@1 ASC, revenueratio@6 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_category@2 ASC, i_class@3 ASC, i_item_id@0 ASC, i_item_desc@1 ASC, revenueratio@6 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_category@2 as i_category, i_class@3 as i_class, i_current_price@4 as i_current_price, sum(catalog_sales.cs_ext_sales_price)@5 as itemrevenue, CAST(sum(catalog_sales.cs_ext_sales_price)@5 AS Float64) * 100 / CAST(sum(sum(catalog_sales.cs_ext_sales_price)) PARTITION BY [item.i_class] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@6 AS Float64) as revenueratio] + │ WindowAggExec: wdw=[sum(sum(catalog_sales.cs_ext_sales_price)) PARTITION BY [item.i_class] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(sum(catalog_sales.cs_ext_sales_price)) PARTITION BY [item.i_class] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Decimal128(27, 2), nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + │ SortExec: expr=[i_class@3 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_class@3], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_category@2 as i_category, i_class@3 as i_class, i_current_price@4 as i_current_price], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, i_category@2, i_class@3, i_current_price@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id, i_item_desc@2 as i_item_desc, i_category@5 as i_category, i_class@4 as i_class, i_current_price@3 as i_current_price], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_ext_sales_price@3, i_item_id@4, i_item_desc@5, i_current_price@6, i_class@7, i_category@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@5 as cs_sold_date_sk, cs_ext_sales_price@6 as cs_ext_sales_price, i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price, i_class@3 as i_class, i_category@4 as i_category] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_item_id@1, i_item_desc@2, i_current_price@3, i_class@4, i_category@5, cs_sold_date_sk@6, cs_ext_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 1999-02-22 AND d_date@1 <= 1999-03-24, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 1999-02-22 AND d_date@1 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_category@5 = Sports OR i_category@5 = Books OR i_category@5 = Home + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@5 = Sports OR i_category@5 = Books OR i_category@5 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_21() -> Result<()> { + let display = test_tpcds_query(21).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [w_warehouse_name@0 ASC, i_item_id@1 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[w_warehouse_name@0 ASC, i_item_id@1 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[w_warehouse_name@0 as w_warehouse_name, i_item_id@1 as i_item_id, sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)@2 as inv_before, sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)@3 as inv_after] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: __common_expr_4@0 >= 0.6666666666666666 AND __common_expr_4@0 <= 1.5, projection=[w_warehouse_name@1, i_item_id@2, sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)@3, sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)@4] + │ ProjectionExec: expr=[CASE WHEN sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)@2 > 0 THEN sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)@3 / sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)@2 END as __common_expr_4, w_warehouse_name@0 as w_warehouse_name, i_item_id@1 as i_item_id, sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)@2 as sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END), sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)@3 as sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)] + │ AggregateExec: mode=FinalPartitioned, gby=[w_warehouse_name@0 as w_warehouse_name, i_item_id@1 as i_item_id], aggr=[sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END), sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([w_warehouse_name@0, i_item_id@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[w_warehouse_name@2 as w_warehouse_name, i_item_id@3 as i_item_id], aggr=[sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END), sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)] + │ ProjectionExec: expr=[d_date@0 as __common_expr_2, inv_quantity_on_hand@1 as inv_quantity_on_hand, w_warehouse_name@2 as w_warehouse_name, i_item_id@3 as i_item_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, inv_date_sk@0)], projection=[d_date@1, inv_quantity_on_hand@3, w_warehouse_name@4, i_item_id@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[inv_date_sk@1 as inv_date_sk, inv_quantity_on_hand@2 as inv_quantity_on_hand, w_warehouse_name@3 as w_warehouse_name, i_item_id@0 as i_item_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, inv_item_sk@1)], projection=[i_item_id@1, inv_date_sk@2, inv_quantity_on_hand@4, w_warehouse_name@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[inv_date_sk@1 as inv_date_sk, inv_item_sk@2 as inv_item_sk, inv_quantity_on_hand@3 as inv_quantity_on_hand, w_warehouse_name@0 as w_warehouse_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(w_warehouse_sk@0, inv_warehouse_sk@2)], projection=[w_warehouse_name@1, inv_date_sk@2, inv_item_sk@3, inv_quantity_on_hand@5] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/warehouse/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet]]}, projection=[w_warehouse_sk, w_warehouse_name], file_type=parquet + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-02-10 AND d_date@1 <= 2000-04-10 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-02-10 AND d_date@1 <= 2000-04-10, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-10 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-10, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_current_price@2 >= Some(99),4,2 AND i_current_price@2 <= Some(149),4,2, projection=[i_item_sk@0, i_item_id@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_current_price], file_type=parquet, predicate=i_current_price@2 >= Some(99),4,2 AND i_current_price@2 <= Some(149),4,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(99),4,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(149),4,2, required_guarantees=[] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_22() -> Result<()> { + let display = test_tpcds_query(22).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [qoh@4 ASC, i_product_name@0 ASC, i_brand@1 ASC, i_class@2 ASC, i_category@3 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[qoh@4 ASC, i_product_name@0 ASC, i_brand@1 ASC, i_class@2 ASC, i_category@3 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_product_name@0 as i_product_name, i_brand@1 as i_brand, i_class@2 as i_class, i_category@3 as i_category, avg(inventory.inv_quantity_on_hand)@5 as qoh] + │ AggregateExec: mode=FinalPartitioned, gby=[i_product_name@0 as i_product_name, i_brand@1 as i_brand, i_class@2 as i_class, i_category@3 as i_category, __grouping_id@4 as __grouping_id], aggr=[avg(inventory.inv_quantity_on_hand)] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_product_name@0, i_brand@1, i_class@2, i_category@3, __grouping_id@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[(NULL as i_product_name, NULL as i_brand, NULL as i_class, NULL as i_category), (i_product_name@4 as i_product_name, NULL as i_brand, NULL as i_class, NULL as i_category), (i_product_name@4 as i_product_name, i_brand@1 as i_brand, NULL as i_class, NULL as i_category), (i_product_name@4 as i_product_name, i_brand@1 as i_brand, i_class@2 as i_class, NULL as i_category), (i_product_name@4 as i_product_name, i_brand@1 as i_brand, i_class@2 as i_class, i_category@3 as i_category)], aggr=[avg(inventory.inv_quantity_on_hand)] + │ ProjectionExec: expr=[inv_quantity_on_hand@4 as inv_quantity_on_hand, i_brand@0 as i_brand, i_class@1 as i_class, i_category@2 as i_category, i_product_name@3 as i_product_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, inv_item_sk@0)], projection=[i_brand@1, i_class@2, i_category@3, i_product_name@4, inv_quantity_on_hand@6] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=1 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_product_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([inv_item_sk@0], 6), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, inv_date_sk@0)], projection=[inv_item_sk@2, inv_quantity_on_hand@3] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/inventory/part-3.parquet:..]]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_23() -> Result<()> { + let display = test_tpcds_query(23).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c_last_name@0 ASC, c_first_name@1 ASC, sales@2 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[c_last_name@0 ASC, c_first_name@1 ASC, sales@2 ASC], preserve_partitioning=[true] + │ InterleaveExec + │ ProjectionExec: expr=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)@2 as sales] + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name], aggr=[sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@3 as c_last_name, c_first_name@2 as c_first_name], aggr=[sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cs_bill_customer_sk@0, CAST(best_ss_customer.c_customer_sk AS Float64)@1)], projection=[cs_quantity@1, cs_list_price@2, c_first_name@3, c_last_name@4] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@0], 3), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@1, item_sk@0)], projection=[cs_bill_customer_sk@0, cs_quantity@2, cs_list_price@3, c_first_name@4, c_last_name@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_bill_customer_sk@3, cs_item_sk@4, cs_quantity@5, cs_list_price@6, c_first_name@7, c_last_name@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@2 as cs_sold_date_sk, cs_bill_customer_sk@3 as cs_bill_customer_sk, cs_item_sk@4 as cs_item_sk, cs_quantity@5 as cs_quantity, cs_list_price@6 as cs_list_price, c_first_name@0 as c_first_name, c_last_name@1 as c_last_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@3, cs_bill_customer_sk@1)], projection=[c_first_name@1, c_last_name@2, cs_sold_date_sk@4, cs_bill_customer_sk@5, cs_item_sk@6, cs_quantity@7, cs_list_price@8] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[i_item_sk@0 as item_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: count(Int64(1))@1 > 4, projection=[i_item_sk@0] + │ ProjectionExec: expr=[i_item_sk@1 as i_item_sk, count(Int64(1))@3 as count(Int64(1))] + │ AggregateExec: mode=FinalPartitioned, gby=[itemdesc@0 as itemdesc, i_item_sk@1 as i_item_sk, d_date@2 as d_date], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([itemdesc@0, i_item_sk@1, d_date@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[itemdesc@1 as itemdesc, i_item_sk@2 as i_item_sk, d_date@0 as d_date], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@1)], projection=[d_date@1, itemdesc@2, i_item_sk@3] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_date@1, ss_item_sk@4] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[substr(i_item_desc@1, 1, 30) as itemdesc, i_item_sk@0 as i_item_sk] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(best_ss_customer.c_customer_sk AS Float64)@1], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, CAST(c_customer_sk@0 AS Float64) as CAST(best_ss_customer.c_customer_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: sum(store_sales.ss_quantity * store_sales.ss_sales_price)@1 > 0.5 * max(max_store_sales.tpcds_cmax)@2, projection=[c_customer_sk@0] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price), max(max_store_sales.tpcds_cmax)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_sk@2 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price), max(max_store_sales.tpcds_cmax)] + │ ProjectionExec: expr=[ss_quantity@1 as ss_quantity, ss_sales_price@2 as ss_sales_price, c_customer_sk@3 as c_customer_sk, tpcds_cmax@0 as tpcds_cmax] + │ CrossJoinExec + │ ProjectionExec: expr=[max(sq2.csales)@0 as tpcds_cmax] + │ AggregateExec: mode=Final, gby=[], aggr=[max(sq2.csales)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[max(sq2.csales)] + │ ProjectionExec: expr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)@1 as csales] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_sk@2 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_quantity@3, ss_sales_price@4, c_customer_sk@5] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@1 as ss_sold_date_sk, ss_quantity@2 as ss_quantity, ss_sales_price@3 as ss_sales_price, c_customer_sk@0 as c_customer_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@1, ss_customer_sk@1)], projection=[c_customer_sk@0, ss_sold_date_sk@2, ss_quantity@4, ss_sales_price@5] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[ss_quantity@1 as ss_quantity, ss_sales_price@2 as ss_sales_price, c_customer_sk@0 as c_customer_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@1, ss_customer_sk@0)], projection=[c_customer_sk@0, ss_quantity@3, ss_sales_price@4] + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, sum(web_sales.ws_quantity * web_sales.ws_list_price)@2 as sales] + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name], aggr=[sum(web_sales.ws_quantity * web_sales.ws_list_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@3 as c_last_name, c_first_name@2 as c_first_name], aggr=[sum(web_sales.ws_quantity * web_sales.ws_list_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ws_bill_customer_sk@0, CAST(best_ss_customer.c_customer_sk AS Float64)@1)], projection=[ws_quantity@1, ws_list_price@2, c_first_name@3, c_last_name@4] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@0], 3), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@0, item_sk@0)], projection=[ws_bill_customer_sk@1, ws_quantity@2, ws_list_price@3, c_first_name@4, c_last_name@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_bill_customer_sk@4, ws_quantity@5, ws_list_price@6, c_first_name@7, c_last_name@8] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@2 as ws_sold_date_sk, ws_item_sk@3 as ws_item_sk, ws_bill_customer_sk@4 as ws_bill_customer_sk, ws_quantity@5 as ws_quantity, ws_list_price@6 as ws_list_price, c_first_name@0 as c_first_name, c_last_name@1 as c_last_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@3, ws_bill_customer_sk@2)], projection=[c_first_name@1, c_last_name@2, ws_sold_date_sk@4, ws_item_sk@5, ws_bill_customer_sk@6, ws_quantity@7, ws_list_price@8] + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[i_item_sk@0 as item_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: count(Int64(1))@1 > 4, projection=[i_item_sk@0] + │ ProjectionExec: expr=[i_item_sk@1 as i_item_sk, count(Int64(1))@3 as count(Int64(1))] + │ AggregateExec: mode=FinalPartitioned, gby=[itemdesc@0 as itemdesc, i_item_sk@1 as i_item_sk, d_date@2 as d_date], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([itemdesc@0, i_item_sk@1, d_date@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[itemdesc@1 as itemdesc, i_item_sk@2 as i_item_sk, d_date@0 as d_date], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@1)], projection=[d_date@1, itemdesc@2, i_item_sk@3] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_date@1, ss_item_sk@4] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[substr(i_item_desc@1, 1, 30) as itemdesc, i_item_sk@0 as i_item_sk] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(best_ss_customer.c_customer_sk AS Float64)@1], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, CAST(c_customer_sk@0 AS Float64) as CAST(best_ss_customer.c_customer_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: sum(store_sales.ss_quantity * store_sales.ss_sales_price)@1 > 0.5 * max(max_store_sales.tpcds_cmax)@2, projection=[c_customer_sk@0] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price), max(max_store_sales.tpcds_cmax)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_sk@2 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price), max(max_store_sales.tpcds_cmax)] + │ ProjectionExec: expr=[ss_quantity@1 as ss_quantity, ss_sales_price@2 as ss_sales_price, c_customer_sk@3 as c_customer_sk, tpcds_cmax@0 as tpcds_cmax] + │ CrossJoinExec + │ ProjectionExec: expr=[max(sq2.csales)@0 as tpcds_cmax] + │ AggregateExec: mode=Final, gby=[], aggr=[max(sq2.csales)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[max(sq2.csales)] + │ ProjectionExec: expr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)@1 as csales] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_sk@2 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_quantity@3, ss_sales_price@4, c_customer_sk@5] + │ CoalescePartitionsExec + │ [Stage 14] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@1 as ss_sold_date_sk, ss_quantity@2 as ss_quantity, ss_sales_price@3 as ss_sales_price, c_customer_sk@0 as c_customer_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@1, ss_customer_sk@1)], projection=[c_customer_sk@0, ss_sold_date_sk@2, ss_quantity@4, ss_sales_price@5] + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[ss_quantity@1 as ss_quantity, ss_sales_price@2 as ss_sales_price, c_customer_sk@0 as c_customer_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@1, ss_customer_sk@0)], projection=[c_customer_sk@0, ss_quantity@3, ss_sales_price@4] + │ [Stage 17] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 18] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2000 AND d_moy@2 = 2, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2000 AND d_moy@2 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@3], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@2 IN (SET) ([2000, 2001, 2002, 2003]), projection=[d_date_sk@0, d_date@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_year], file_type=parquet, predicate=d_year@2 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 IN (SET) ([2000, 2001, 2002, 2003]), projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@1], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@1], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_customer_sk@0], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2000 AND d_moy@2 = 2, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2000 AND d_moy@2 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@3], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@2 IN (SET) ([2000, 2001, 2002, 2003]), projection=[d_date_sk@0, d_date@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_year], file_type=parquet, predicate=d_year@2 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 IN (SET) ([2000, 2001, 2002, 2003]), projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@1], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 17 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@1], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 18 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_customer_sk@0], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_24() -> Result<()> { + let display = test_tpcds_query(24).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortExec: expr=[c_last_name@0 ASC NULLS LAST, c_first_name@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST], preserve_partitioning=[false] + │ ProjectionExec: expr=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, s_store_name@2 as s_store_name, sum(ssales.netpaid)@3 as paid] + │ NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 > Float64(0.05) * avg(ssales.netpaid)@0, projection=[c_last_name@0, c_first_name@1, s_store_name@2, sum(ssales.netpaid)@3, Float64(0.05) * avg(ssales.netpaid)@5] + │ ProjectionExec: expr=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, s_store_name@2 as s_store_name, sum(ssales.netpaid)@3 as sum(ssales.netpaid), CAST(sum(ssales.netpaid)@3 AS Decimal128(38, 15)) as join_proj_push_down_1] + │ CoalescePartitionsExec + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, s_store_name@2 as s_store_name], aggr=[sum(ssales.netpaid)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, s_store_name@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, s_store_name@2 as s_store_name], aggr=[sum(ssales.netpaid)] + │ ProjectionExec: expr=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, s_store_name@2 as s_store_name, sum(store_sales.ss_net_paid)@10 as netpaid] + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, s_store_name@2 as s_store_name, ca_state@3 as ca_state, s_state@4 as s_state, i_color@5 as i_color, i_current_price@6 as i_current_price, i_manager_id@7 as i_manager_id, i_units@8 as i_units, i_size@9 as i_size], aggr=[sum(store_sales.ss_net_paid)], ordering_mode=PartiallySorted([5]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, s_store_name@2, ca_state@3, s_state@4, i_color@5, i_current_price@6, i_manager_id@7, i_units@8, i_size@9], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@9 as c_last_name, c_first_name@8 as c_first_name, s_store_name@1 as s_store_name, ca_state@10 as ca_state, s_state@2 as s_state, i_color@5 as i_color, i_current_price@3 as i_current_price, i_manager_id@7 as i_manager_id, i_units@6 as i_units, i_size@4 as i_size], aggr=[sum(store_sales.ss_net_paid)], ordering_mode=PartiallySorted([5]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@9, ca_address_sk@0), (s_zip@3, ca_zip@2)], filter=c_birth_country@0 != CAST(upper(ca_country@1) AS Utf8View), projection=[ss_net_paid@0, s_store_name@1, s_state@2, i_current_price@4, i_size@5, i_color@6, i_units@7, i_manager_id@8, c_first_name@10, c_last_name@11, ca_state@14] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@5)], projection=[ss_net_paid@1, s_store_name@2, s_state@3, s_zip@4, i_current_price@5, i_size@6, i_color@7, i_units@8, i_manager_id@9, c_current_addr_sk@11, c_first_name@12, c_last_name@13, c_birth_country@14] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_customer_sk@1, ss_net_paid@2, s_store_name@3, s_state@4, s_zip@5, i_current_price@7, i_size@8, i_color@9, i_units@10, i_manager_id@11] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_item_sk@3 as ss_item_sk, ss_customer_sk@4 as ss_customer_sk, ss_net_paid@5 as ss_net_paid, s_store_name@0 as s_store_name, s_state@1 as s_state, s_zip@2 as s_zip] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@4, ss_store_sk@2)], projection=[s_store_name@1, s_state@2, s_zip@3, ss_item_sk@5, ss_customer_sk@6, ss_net_paid@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_ticket_number@1, ss_ticket_number@3), (sr_item_sk@0, ss_item_sk@0)], projection=[ss_item_sk@2, ss_customer_sk@3, ss_store_sk@4, ss_net_paid@6] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_color@3 = peach + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet, predicate=i_color@3 = peach AND DynamicFilter [ empty ], pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= peach AND peach <= i_color_max@1, required_guarantees=[i_color in (peach)] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_birth_country@4 as c_birth_country, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country], file_type=parquet + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state, ca_zip, ca_country], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[CAST(0.05 * CAST(avg(ssales.netpaid)@0 AS Float64) AS Decimal128(38, 15)) as Float64(0.05) * avg(ssales.netpaid)] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(ssales.netpaid)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(ssales.netpaid)] + │ ProjectionExec: expr=[sum(store_sales.ss_net_paid)@10 as netpaid] + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, s_store_name@2 as s_store_name, ca_state@3 as ca_state, s_state@4 as s_state, i_color@5 as i_color, i_current_price@6 as i_current_price, i_manager_id@7 as i_manager_id, i_units@8 as i_units, i_size@9 as i_size], aggr=[sum(store_sales.ss_net_paid)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, s_store_name@2, ca_state@3, s_state@4, i_color@5, i_current_price@6, i_manager_id@7, i_units@8, i_size@9], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@9 as c_last_name, c_first_name@8 as c_first_name, s_store_name@1 as s_store_name, ca_state@10 as ca_state, s_state@2 as s_state, i_color@5 as i_color, i_current_price@3 as i_current_price, i_manager_id@7 as i_manager_id, i_units@6 as i_units, i_size@4 as i_size], aggr=[sum(store_sales.ss_net_paid)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@9, ca_address_sk@0), (s_zip@3, ca_zip@2)], filter=c_birth_country@0 != CAST(upper(ca_country@1) AS Utf8View), projection=[ss_net_paid@0, s_store_name@1, s_state@2, i_current_price@4, i_size@5, i_color@6, i_units@7, i_manager_id@8, c_first_name@10, c_last_name@11, ca_state@14] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@5)], projection=[ss_net_paid@1, s_store_name@2, s_state@3, s_zip@4, i_current_price@5, i_size@6, i_color@7, i_units@8, i_manager_id@9, c_current_addr_sk@11, c_first_name@12, c_last_name@13, c_birth_country@14] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_customer_sk@1, ss_net_paid@2, s_store_name@3, s_state@4, s_zip@5, i_current_price@7, i_size@8, i_color@9, i_units@10, i_manager_id@11] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_item_sk@3 as ss_item_sk, ss_customer_sk@4 as ss_customer_sk, ss_net_paid@5 as ss_net_paid, s_store_name@0 as s_store_name, s_state@1 as s_state, s_zip@2 as s_zip] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@4, ss_store_sk@2)], projection=[s_store_name@1, s_state@2, s_zip@3, ss_item_sk@5, ss_customer_sk@6, ss_net_paid@8] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_ticket_number@1, ss_ticket_number@3), (sr_item_sk@0, ss_item_sk@0)], projection=[ss_item_sk@2, ss_customer_sk@3, ss_store_sk@4, ss_net_paid@6] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_birth_country@4 as c_birth_country, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country], file_type=parquet + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state, ca_zip, ca_country], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_name@1 as s_store_name, s_state@2 as s_state, s_zip@3 as s_zip, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_market_id@2 = 8, projection=[s_store_sk@0, s_store_name@1, s_state@3, s_zip@4] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_market_id, s_state, s_zip], file_type=parquet, predicate=s_market_id@2 = 8, pruning_predicate=s_market_id_null_count@2 != row_count@3 AND s_market_id_min@0 <= 8 AND 8 <= s_market_id_max@1, required_guarantees=[s_market_id in (8)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_ticket_number@3, ss_item_sk@0], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_name@1 as s_store_name, s_state@2 as s_state, s_zip@3 as s_zip, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_market_id@2 = 8, projection=[s_store_sk@0, s_store_name@1, s_state@3, s_zip@4] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_market_id, s_state, s_zip], file_type=parquet, predicate=s_market_id@2 = 8, pruning_predicate=s_market_id_null_count@2 != row_count@3 AND s_market_id_min@0 <= 8 AND 8 <= s_market_id_max@1, required_guarantees=[s_market_id in (8)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_ticket_number@3, ss_item_sk@0], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_25() -> Result<()> { + let display = test_tpcds_query(25).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, s_store_id@2 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, s_store_id@2 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, s_store_id@2 as s_store_id, s_store_name@3 as s_store_name, sum(store_sales.ss_net_profit)@4 as store_sales_profit, sum(store_returns.sr_net_loss)@5 as store_returns_loss, sum(catalog_sales.cs_net_profit)@6 as catalog_sales_profit] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, s_store_id@2 as s_store_id, s_store_name@3 as s_store_name], aggr=[sum(store_sales.ss_net_profit), sum(store_returns.sr_net_loss), sum(catalog_sales.cs_net_profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, s_store_id@2, s_store_name@3], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@5 as i_item_id, i_item_desc@6 as i_item_desc, s_store_id@3 as s_store_id, s_store_name@4 as s_store_name], aggr=[sum(store_sales.ss_net_profit), sum(store_returns.sr_net_loss), sum(catalog_sales.cs_net_profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_net_profit@1, sr_net_loss@2, cs_net_profit@3, s_store_id@4, s_store_name@5, i_item_id@7, i_item_desc@8] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_store_sk@1, CAST(store.s_store_sk AS Float64)@3)], projection=[ss_item_sk@0, ss_net_profit@2, sr_net_loss@3, cs_net_profit@4, s_store_id@6, s_store_name@7] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_sold_date_sk@4, CAST(d3.d_date_sk AS Float64)@1)], projection=[ss_item_sk@0, ss_store_sk@1, ss_net_profit@2, sr_net_loss@3, cs_net_profit@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(sr_returned_date_sk@3, CAST(d2.d_date_sk AS Float64)@1)], projection=[ss_item_sk@0, ss_store_sk@1, ss_net_profit@2, sr_net_loss@4, cs_sold_date_sk@5, cs_net_profit@6] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_net_profit@5, sr_returned_date_sk@6, sr_net_loss@7, cs_sold_date_sk@8, cs_net_profit@9] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@2 as ss_sold_date_sk, ss_item_sk@3 as ss_item_sk, ss_store_sk@4 as ss_store_sk, ss_net_profit@5 as ss_net_profit, sr_returned_date_sk@6 as sr_returned_date_sk, sr_net_loss@7 as sr_net_loss, cs_sold_date_sk@0 as cs_sold_date_sk, cs_net_profit@1 as cs_net_profit] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cs_bill_customer_sk@1, sr_customer_sk@6), (cs_item_sk@2, sr_item_sk@5)], projection=[cs_sold_date_sk@0, cs_net_profit@3, ss_sold_date_sk@4, ss_item_sk@5, ss_store_sk@6, ss_net_profit@7, sr_returned_date_sk@8, sr_net_loss@11] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_moy@2 >= 4 AND d_moy@2 <= 10 AND d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 >= 4 AND d_moy@2 <= 10 AND d_year@1 = 2001, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 4 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 10 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_year in (2001)] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_moy@2 >= 4 AND d_moy@2 <= 10 AND d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 >= 4 AND d_moy@2 <= 10 AND d_year@1 = 2001, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 4 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 10 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_year in (2001)] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_id@1 as s_store_id, s_store_name@2 as s_store_name, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id, s_store_name], file_type=parquet + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_moy@2 = 4 AND d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 4 AND d_year@1 = 2001, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 4 AND 4 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_moy in (4), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1, cs_item_sk@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_net_profit], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_customer_sk@6, sr_item_sk@5], 3), input_partitions=3 + │ ProjectionExec: expr=[ss_sold_date_sk@4 as ss_sold_date_sk, ss_item_sk@5 as ss_item_sk, ss_store_sk@6 as ss_store_sk, ss_net_profit@7 as ss_net_profit, sr_returned_date_sk@0 as sr_returned_date_sk, sr_item_sk@1 as sr_item_sk, sr_customer_sk@2 as sr_customer_sk, sr_net_loss@3 as sr_net_loss] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_customer_sk@2, ss_customer_sk@2), (sr_item_sk@1, ss_item_sk@1), (sr_ticket_number@3, ss_ticket_number@4)], projection=[sr_returned_date_sk@0, sr_item_sk@1, sr_customer_sk@2, sr_net_loss@4, ss_sold_date_sk@5, ss_item_sk@6, ss_store_sk@8, ss_net_profit@10] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_customer_sk@2, sr_item_sk@1, sr_ticket_number@3], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_customer_sk@2, ss_item_sk@1, ss_ticket_number@4], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_26() -> Result<()> { + let display = test_tpcds_query(26).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, avg(catalog_sales.cs_quantity)@1 as agg1, avg(catalog_sales.cs_list_price)@2 as agg2, avg(catalog_sales.cs_coupon_amt)@3 as agg3, avg(catalog_sales.cs_sales_price)@4 as agg4] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[avg(catalog_sales.cs_quantity), avg(catalog_sales.cs_list_price), avg(catalog_sales.cs_coupon_amt), avg(catalog_sales.cs_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@4 as i_item_id], aggr=[avg(catalog_sales.cs_quantity), avg(catalog_sales.cs_list_price), avg(catalog_sales.cs_coupon_amt), avg(catalog_sales.cs_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(promotion.p_promo_sk AS Float64)@1, cs_promo_sk@0)], projection=[cs_quantity@3, cs_list_price@4, cs_sales_price@5, cs_coupon_amt@6, i_item_id@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, i_item_sk@0)], projection=[cs_promo_sk@1, cs_quantity@2, cs_list_price@3, cs_sales_price@4, cs_coupon_amt@5, i_item_id@7] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_item_sk@3, cs_promo_sk@4, cs_quantity@5, cs_list_price@6, cs_sales_price@7, cs_coupon_amt@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, cs_bill_cdemo_sk@1)], projection=[cs_sold_date_sk@2, cs_item_sk@4, cs_promo_sk@5, cs_quantity@6, cs_list_price@7, cs_sales_price@8, cs_coupon_amt@9] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: p_channel_email@1 = N OR p_channel_event@2 = N, projection=[p_promo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, p_channel_email, p_channel_event], file_type=parquet, predicate=p_channel_email@1 = N OR p_channel_event@2 = N, pruning_predicate=p_channel_email_null_count@2 != row_count@3 AND p_channel_email_min@0 <= N AND N <= p_channel_email_max@1 OR p_channel_event_null_count@6 != row_count@3 AND p_channel_event_min@4 <= N AND N <= p_channel_event_max@5, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=3 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_bill_cdemo_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_cdemo_sk, cs_item_sk, cs_promo_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_27() -> Result<()> { + let display = test_tpcds_query(27).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_item_id@0 ASC, s_state@1 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC], preserve_partitioning=[true] + │ UnionExec + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, CAST(s_state@1 AS Utf8) as s_state, 0 as g_state, avg(results.agg1)@2 as agg1, avg(results.agg2)@3 as agg2, avg(results.agg3)@4 as agg3, avg(results.agg4)@5 as agg4] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, s_state@1 as s_state], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)], ordering_mode=PartiallySorted([1]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0, s_state@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@0 as i_item_id, s_state@1 as s_state], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)], ordering_mode=PartiallySorted([1]) + │ ProjectionExec: expr=[i_item_id@5 as i_item_id, s_state@4 as s_state, ss_quantity@0 as agg1, ss_list_price@1 as agg2, ss_coupon_amt@3 as agg3, ss_sales_price@2 as agg4] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_quantity@1, ss_list_price@2, ss_sales_price@3, ss_coupon_amt@4, s_state@5, i_item_id@7] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_quantity@2 as ss_quantity, ss_list_price@3 as ss_list_price, ss_sales_price@4 as ss_sales_price, ss_coupon_amt@5 as ss_coupon_amt, s_state@0 as s_state] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, ss_store_sk@1)], projection=[s_state@1, ss_item_sk@3, ss_quantity@5, ss_list_price@6, ss_sales_price@7, ss_coupon_amt@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_quantity@5, ss_list_price@6, ss_sales_price@7, ss_coupon_amt@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, ss_cdemo_sk@2)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_store_sk@5, ss_quantity@6, ss_list_price@7, ss_sales_price@8, ss_coupon_amt@9] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, NULL as s_state, 1 as g_state, avg(results.agg1)@1 as agg1, avg(results.agg2)@2 as agg2, avg(results.agg3)@3 as agg3, avg(results.agg4)@4 as agg4] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@0 as i_item_id], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)] + │ ProjectionExec: expr=[i_item_id@4 as i_item_id, ss_quantity@0 as agg1, ss_list_price@1 as agg2, ss_coupon_amt@3 as agg3, ss_sales_price@2 as agg4] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_quantity@1, ss_list_price@2, ss_sales_price@3, ss_coupon_amt@4, i_item_id@6] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@1)], projection=[ss_item_sk@2, ss_quantity@4, ss_list_price@5, ss_sales_price@6, ss_coupon_amt@7] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_quantity@5, ss_list_price@6, ss_sales_price@7, ss_coupon_amt@8] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, ss_cdemo_sk@2)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_store_sk@5, ss_quantity@6, ss_list_price@7, ss_sales_price@8, ss_coupon_amt@9] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[NULL as i_item_id, NULL as s_state, 1 as g_state, avg(results.agg1)@0 as agg1, avg(results.agg2)@1 as agg2, avg(results.agg3)@2 as agg3, avg(results.agg4)@3 as agg4] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)] + │ ProjectionExec: expr=[ss_quantity@0 as agg1, ss_list_price@1 as agg2, ss_coupon_amt@3 as agg3, ss_sales_price@2 as agg4] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_quantity@1, ss_list_price@2, ss_sales_price@3, ss_coupon_amt@4] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@1)], projection=[ss_item_sk@2, ss_quantity@4, ss_list_price@5, ss_sales_price@6, ss_coupon_amt@7] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_quantity@5, ss_list_price@6, ss_sales_price@7, ss_coupon_amt@8] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, ss_cdemo_sk@2)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_store_sk@5, ss_quantity@6, ss_list_price@7, ss_sales_price@8, ss_coupon_amt@9] + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_state@1 as s_state, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_state@1 = TN + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@1 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=3 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@1 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=3 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@1 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=3 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_28() -> Result<()> { + let display = test_tpcds_query(28).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[b1_lp@3 as b1_lp, b1_cnt@4 as b1_cnt, b1_cntd@5 as b1_cntd, b2_lp@6 as b2_lp, b2_cnt@7 as b2_cnt, b2_cntd@8 as b2_cntd, b3_lp@9 as b3_lp, b3_cnt@10 as b3_cnt, b3_cntd@11 as b3_cntd, b4_lp@12 as b4_lp, b4_cnt@13 as b4_cnt, b4_cntd@14 as b4_cntd, b5_lp@15 as b5_lp, b5_cnt@16 as b5_cnt, b5_cntd@17 as b5_cntd, b6_lp@0 as b6_lp, b6_cnt@1 as b6_cnt, b6_cntd@2 as b6_cntd] + │ GlobalLimitExec: skip=0, fetch=100 + │ CrossJoinExec + │ ProjectionExec: expr=[avg(store_sales.ss_list_price)@0 as b6_lp, count(store_sales.ss_list_price)@1 as b6_cnt, count(DISTINCT store_sales.ss_list_price)@2 as b6_cntd] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[b1_lp@3 as b1_lp, b1_cnt@4 as b1_cnt, b1_cntd@5 as b1_cntd, b2_lp@6 as b2_lp, b2_cnt@7 as b2_cnt, b2_cntd@8 as b2_cntd, b3_lp@9 as b3_lp, b3_cnt@10 as b3_cnt, b3_cntd@11 as b3_cntd, b4_lp@12 as b4_lp, b4_cnt@13 as b4_cnt, b4_cntd@14 as b4_cntd, b5_lp@0 as b5_lp, b5_cnt@1 as b5_cnt, b5_cntd@2 as b5_cntd] + │ CrossJoinExec + │ ProjectionExec: expr=[avg(store_sales.ss_list_price)@0 as b5_lp, count(store_sales.ss_list_price)@1 as b5_cnt, count(DISTINCT store_sales.ss_list_price)@2 as b5_cntd] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[b1_lp@3 as b1_lp, b1_cnt@4 as b1_cnt, b1_cntd@5 as b1_cntd, b2_lp@6 as b2_lp, b2_cnt@7 as b2_cnt, b2_cntd@8 as b2_cntd, b3_lp@9 as b3_lp, b3_cnt@10 as b3_cnt, b3_cntd@11 as b3_cntd, b4_lp@0 as b4_lp, b4_cnt@1 as b4_cnt, b4_cntd@2 as b4_cntd] + │ CrossJoinExec + │ ProjectionExec: expr=[avg(store_sales.ss_list_price)@0 as b4_lp, count(store_sales.ss_list_price)@1 as b4_cnt, count(DISTINCT store_sales.ss_list_price)@2 as b4_cntd] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[b1_lp@3 as b1_lp, b1_cnt@4 as b1_cnt, b1_cntd@5 as b1_cntd, b2_lp@6 as b2_lp, b2_cnt@7 as b2_cnt, b2_cntd@8 as b2_cntd, b3_lp@0 as b3_lp, b3_cnt@1 as b3_cnt, b3_cntd@2 as b3_cntd] + │ CrossJoinExec + │ ProjectionExec: expr=[avg(store_sales.ss_list_price)@0 as b3_lp, count(store_sales.ss_list_price)@1 as b3_cnt, count(DISTINCT store_sales.ss_list_price)@2 as b3_cntd] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CrossJoinExec + │ ProjectionExec: expr=[avg(store_sales.ss_list_price)@0 as b1_lp, count(store_sales.ss_list_price)@1 as b1_cnt, count(DISTINCT store_sales.ss_list_price)@2 as b1_cntd] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[avg(store_sales.ss_list_price)@0 as b2_lp, count(store_sales.ss_list_price)@1 as b2_cnt, count(DISTINCT store_sales.ss_list_price)@2 as b2_cntd] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 26 AND ss_quantity@0 <= 30 AND (ss_list_price@2 >= Some(15400),5,2 AND ss_list_price@2 <= Some(16400),5,2 OR ss_coupon_amt@3 >= Some(732600),7,2 AND ss_coupon_amt@3 <= Some(832600),7,2 OR ss_wholesale_cost@1 >= Some(700),5,2 AND ss_wholesale_cost@1 <= Some(2700),5,2), projection=[ss_list_price@2] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@0 >= 26 AND ss_quantity@0 <= 30 AND (ss_list_price@2 >= Some(15400),5,2 AND ss_list_price@2 <= Some(16400),5,2 OR ss_coupon_amt@3 >= Some(732600),7,2 AND ss_coupon_amt@3 <= Some(832600),7,2 OR ss_wholesale_cost@1 >= Some(700),5,2 AND ss_wholesale_cost@1 <= Some(2700),5,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 26 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 30 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(15400),5,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(16400),5,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(732600),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(832600),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(700),5,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(2700),5,2), required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 21 AND ss_quantity@0 <= 25 AND (ss_list_price@2 >= Some(12200),5,2 AND ss_list_price@2 <= Some(13200),5,2 OR ss_coupon_amt@3 >= Some(83600),7,2 AND ss_coupon_amt@3 <= Some(183600),7,2 OR ss_wholesale_cost@1 >= Some(1700),5,2 AND ss_wholesale_cost@1 <= Some(3700),5,2), projection=[ss_list_price@2] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@0 >= 21 AND ss_quantity@0 <= 25 AND (ss_list_price@2 >= Some(12200),5,2 AND ss_list_price@2 <= Some(13200),5,2 OR ss_coupon_amt@3 >= Some(83600),7,2 AND ss_coupon_amt@3 <= Some(183600),7,2 OR ss_wholesale_cost@1 >= Some(1700),5,2 AND ss_wholesale_cost@1 <= Some(3700),5,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 21 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 25 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(12200),5,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(13200),5,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(83600),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(183600),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(1700),5,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(3700),5,2), required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 16 AND ss_quantity@0 <= 20 AND (ss_list_price@2 >= Some(13500),5,2 AND ss_list_price@2 <= Some(14500),5,2 OR ss_coupon_amt@3 >= Some(607100),7,2 AND ss_coupon_amt@3 <= Some(707100),7,2 OR ss_wholesale_cost@1 >= Some(3800),5,2 AND ss_wholesale_cost@1 <= Some(5800),5,2), projection=[ss_list_price@2] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@0 >= 16 AND ss_quantity@0 <= 20 AND (ss_list_price@2 >= Some(13500),5,2 AND ss_list_price@2 <= Some(14500),5,2 OR ss_coupon_amt@3 >= Some(607100),7,2 AND ss_coupon_amt@3 <= Some(707100),7,2 OR ss_wholesale_cost@1 >= Some(3800),5,2 AND ss_wholesale_cost@1 <= Some(5800),5,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 16 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 20 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(13500),5,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(14500),5,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(607100),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(707100),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(3800),5,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(5800),5,2), required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 11 AND ss_quantity@0 <= 15 AND (ss_list_price@2 >= Some(14200),5,2 AND ss_list_price@2 <= Some(15200),5,2 OR ss_coupon_amt@3 >= Some(1221400),7,2 AND ss_coupon_amt@3 <= Some(1321400),7,2 OR ss_wholesale_cost@1 >= Some(7900),5,2 AND ss_wholesale_cost@1 <= Some(9900),5,2), projection=[ss_list_price@2] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@0 >= 11 AND ss_quantity@0 <= 15 AND (ss_list_price@2 >= Some(14200),5,2 AND ss_list_price@2 <= Some(15200),5,2 OR ss_coupon_amt@3 >= Some(1221400),7,2 AND ss_coupon_amt@3 <= Some(1321400),7,2 OR ss_wholesale_cost@1 >= Some(7900),5,2 AND ss_wholesale_cost@1 <= Some(9900),5,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 11 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 15 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(14200),5,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(15200),5,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(1221400),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(1321400),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(7900),5,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(9900),5,2), required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 0 AND ss_quantity@0 <= 5 AND (ss_list_price@2 >= Some(800),5,2 AND ss_list_price@2 <= Some(1800),5,2 OR ss_coupon_amt@3 >= Some(45900),7,2 AND ss_coupon_amt@3 <= Some(145900),7,2 OR ss_wholesale_cost@1 >= Some(5700),5,2 AND ss_wholesale_cost@1 <= Some(7700),5,2), projection=[ss_list_price@2] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@0 >= 0 AND ss_quantity@0 <= 5 AND (ss_list_price@2 >= Some(800),5,2 AND ss_list_price@2 <= Some(1800),5,2 OR ss_coupon_amt@3 >= Some(45900),7,2 AND ss_coupon_amt@3 <= Some(145900),7,2 OR ss_wholesale_cost@1 >= Some(5700),5,2 AND ss_wholesale_cost@1 <= Some(7700),5,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 0 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 5 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(800),5,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(1800),5,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(45900),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(145900),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(5700),5,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(7700),5,2), required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_quantity@0 >= 6 AND ss_quantity@0 <= 10 AND (ss_list_price@2 >= Some(9000),5,2 AND ss_list_price@2 <= Some(10000),5,2 OR ss_coupon_amt@3 >= Some(232300),7,2 AND ss_coupon_amt@3 <= Some(332300),7,2 OR ss_wholesale_cost@1 >= Some(3100),5,2 AND ss_wholesale_cost@1 <= Some(5100),5,2), projection=[ss_list_price@2] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@0 >= 6 AND ss_quantity@0 <= 10 AND (ss_list_price@2 >= Some(9000),5,2 AND ss_list_price@2 <= Some(10000),5,2 OR ss_coupon_amt@3 >= Some(232300),7,2 AND ss_coupon_amt@3 <= Some(332300),7,2 OR ss_wholesale_cost@1 >= Some(3100),5,2 AND ss_wholesale_cost@1 <= Some(5100),5,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 6 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 10 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(9000),5,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(10000),5,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(232300),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(332300),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(3100),5,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(5100),5,2), required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_29() -> Result<()> { + let display = test_tpcds_query(29).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, s_store_id@2 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, s_store_id@2 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, s_store_id@2 as s_store_id, s_store_name@3 as s_store_name, sum(store_sales.ss_quantity)@4 as store_sales_quantity, sum(store_returns.sr_return_quantity)@5 as store_returns_quantity, sum(catalog_sales.cs_quantity)@6 as catalog_sales_quantity] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, s_store_id@2 as s_store_id, s_store_name@3 as s_store_name], aggr=[sum(store_sales.ss_quantity), sum(store_returns.sr_return_quantity), sum(catalog_sales.cs_quantity)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, s_store_id@2, s_store_name@3], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@5 as i_item_id, i_item_desc@6 as i_item_desc, s_store_id@3 as s_store_id, s_store_name@4 as s_store_name], aggr=[sum(store_sales.ss_quantity), sum(store_returns.sr_return_quantity), sum(catalog_sales.cs_quantity)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_quantity@1, sr_return_quantity@2, cs_quantity@3, s_store_id@4, s_store_name@5, i_item_id@7, i_item_desc@8] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_store_sk@1, CAST(store.s_store_sk AS Float64)@3)], projection=[ss_item_sk@0, ss_quantity@2, sr_return_quantity@3, cs_quantity@4, s_store_id@6, s_store_name@7] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_sold_date_sk@4, CAST(d3.d_date_sk AS Float64)@1)], projection=[ss_item_sk@0, ss_store_sk@1, ss_quantity@2, sr_return_quantity@3, cs_quantity@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d2.d_date_sk AS Float64)@1, sr_returned_date_sk@3)], projection=[ss_item_sk@2, ss_store_sk@3, ss_quantity@4, sr_return_quantity@6, cs_sold_date_sk@7, cs_quantity@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_quantity@5, sr_returned_date_sk@6, sr_return_quantity@7, cs_sold_date_sk@8, cs_quantity@9] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@2 as ss_sold_date_sk, ss_item_sk@3 as ss_item_sk, ss_store_sk@4 as ss_store_sk, ss_quantity@5 as ss_quantity, sr_returned_date_sk@6 as sr_returned_date_sk, sr_return_quantity@7 as sr_return_quantity, cs_sold_date_sk@0 as cs_sold_date_sk, cs_quantity@1 as cs_quantity] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cs_bill_customer_sk@1, sr_customer_sk@6), (cs_item_sk@2, sr_item_sk@5)], projection=[cs_sold_date_sk@0, cs_quantity@3, ss_sold_date_sk@4, ss_item_sk@5, ss_store_sk@6, ss_quantity@7, sr_returned_date_sk@8, sr_return_quantity@11] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (1999, 2000, 2001)] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_id@1 as s_store_id, s_store_name@2 as s_store_name, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id, s_store_name], file_type=parquet + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_moy@2 >= 9 AND d_moy@2 <= 12 AND d_year@1 = 1999, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 >= 9 AND d_moy@2 <= 12 AND d_year@1 = 1999, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 9 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 12 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_year in (1999)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_moy@2 = 9 AND d_year@1 = 1999, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 9 AND d_year@1 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 9 AND 9 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (9), d_year in (1999)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1, cs_item_sk@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_customer_sk@6, sr_item_sk@5], 3), input_partitions=3 + │ ProjectionExec: expr=[ss_sold_date_sk@4 as ss_sold_date_sk, ss_item_sk@5 as ss_item_sk, ss_store_sk@6 as ss_store_sk, ss_quantity@7 as ss_quantity, sr_returned_date_sk@0 as sr_returned_date_sk, sr_item_sk@1 as sr_item_sk, sr_customer_sk@2 as sr_customer_sk, sr_return_quantity@3 as sr_return_quantity] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_customer_sk@2, ss_customer_sk@2), (sr_item_sk@1, ss_item_sk@1), (sr_ticket_number@3, ss_ticket_number@4)], projection=[sr_returned_date_sk@0, sr_item_sk@1, sr_customer_sk@2, sr_return_quantity@4, ss_sold_date_sk@5, ss_item_sk@6, ss_store_sk@8, ss_quantity@10] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_customer_sk@2, sr_item_sk@1, sr_ticket_number@3], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_customer_sk@2, ss_item_sk@1, ss_ticket_number@4], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + #[ignore = "Fails with column 'c_last_review_date_sk' not found"] + async fn test_tpcds_30() -> Result<()> { + let display = test_tpcds_query(30).await?; + assert_snapshot!(display, @r""); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_31() -> Result<()> { + let display = test_tpcds_query(31).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [ca_county@0 ASC NULLS LAST] + │ SortExec: expr=[ca_county@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[ca_county@2 as ca_county, d_year@3 as d_year, __common_expr_1@0 / CAST(web_sales@6 AS Float64) as web_q1_q2_increase, __common_expr_2@1 / CAST(store_sales@4 AS Float64) as store_q1_q2_increase, CAST(web_sales@7 AS Float64) / __common_expr_1@0 as web_q2_q3_increase, CAST(store_sales@5 AS Float64) / __common_expr_2@1 as store_q2_q3_increase] + │ ProjectionExec: expr=[CAST(web_sales@6 AS Float64) as __common_expr_1, CAST(store_sales@3 AS Float64) as __common_expr_2, ca_county@0 as ca_county, d_year@1 as d_year, store_sales@2 as store_sales, store_sales@4 as store_sales, web_sales@5 as web_sales, web_sales@7 as web_sales] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_county@5, ca_county@0)], filter=CASE WHEN web_sales@2 > Some(0),17,2 THEN CAST(web_sales@3 AS Float64) / CAST(web_sales@2 AS Float64) END > CASE WHEN store_sales@0 > Some(0),17,2 THEN CAST(store_sales@1 AS Float64) / CAST(store_sales@0 AS Float64) END, projection=[ca_county@0, d_year@1, store_sales@2, store_sales@3, store_sales@4, web_sales@6, web_sales@7, web_sales@9] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_county@5, ca_county@0)], filter=CASE WHEN web_sales@2 > Some(0),17,2 THEN CAST(web_sales@3 AS Float64) / CAST(web_sales@2 AS Float64) END > CASE WHEN store_sales@0 > Some(0),17,2 THEN CAST(store_sales@1 AS Float64) / CAST(store_sales@0 AS Float64) END, projection=[ca_county@0, d_year@1, store_sales@2, store_sales@3, store_sales@4, ca_county@5, web_sales@6, web_sales@8] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ca_county@2 as ca_county, d_year@3 as d_year, store_sales@4 as store_sales, store_sales@5 as store_sales, store_sales@6 as store_sales, ca_county@0 as ca_county, web_sales@1 as web_sales] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_county@0, ca_county@0)] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ca_county@0 as ca_county, sum(web_sales.ws_ext_sales_price)@3 as web_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[ca_county@0 as ca_county, d_qoy@1 as d_qoy, d_year@2 as d_year], aggr=[sum(web_sales.ws_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ca_county@0, d_qoy@1, d_year@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_county@3 as ca_county, d_qoy@2 as d_qoy, d_year@1 as d_year], aggr=[sum(web_sales.ws_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_bill_addr_sk@0, CAST(customer_address.ca_address_sk AS Float64)@2)], projection=[ws_ext_sales_price@1, d_year@2, d_qoy@3, ca_county@5] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ws_bill_addr_sk@2 as ws_bill_addr_sk, ws_ext_sales_price@3 as ws_ext_sales_price, d_year@0 as d_year, d_qoy@1 as d_qoy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ws_sold_date_sk@0)], projection=[d_year@1, d_qoy@2, ws_bill_addr_sk@5, ws_ext_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_county@1 as ca_county, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_county], file_type=parquet + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_county@3, ca_county@0)], projection=[ca_county@0, d_year@1, store_sales@2, store_sales@4, store_sales@6] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_county@0, ca_county@0)] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ca_county@0 as ca_county, d_year@2 as d_year, sum(store_sales.ss_ext_sales_price)@3 as store_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[ca_county@0 as ca_county, d_qoy@1 as d_qoy, d_year@2 as d_year], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ca_county@0, d_qoy@1, d_year@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_county@3 as ca_county, d_qoy@2 as d_qoy, d_year@1 as d_year], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@0, CAST(customer_address.ca_address_sk AS Float64)@2)], projection=[ss_ext_sales_price@1, d_year@2, d_qoy@3, ca_county@5] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_addr_sk@2 as ss_addr_sk, ss_ext_sales_price@3 as ss_ext_sales_price, d_year@0 as d_year, d_qoy@1 as d_qoy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@0)], projection=[d_year@1, d_qoy@2, ss_addr_sk@5, ss_ext_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_county@1 as ca_county, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_county], file_type=parquet + │ ProjectionExec: expr=[ca_county@0 as ca_county, sum(store_sales.ss_ext_sales_price)@3 as store_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[ca_county@0 as ca_county, d_qoy@1 as d_qoy, d_year@2 as d_year], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ca_county@0, d_qoy@1, d_year@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_county@3 as ca_county, d_qoy@2 as d_qoy, d_year@1 as d_year], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@0, CAST(customer_address.ca_address_sk AS Float64)@2)], projection=[ss_ext_sales_price@1, d_year@2, d_qoy@3, ca_county@5] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_addr_sk@2 as ss_addr_sk, ss_ext_sales_price@3 as ss_ext_sales_price, d_year@0 as d_year, d_qoy@1 as d_qoy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@0)], projection=[d_year@1, d_qoy@2, ss_addr_sk@5, ss_ext_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_county@1 as ca_county, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_county], file_type=parquet + │ ProjectionExec: expr=[ca_county@0 as ca_county, sum(store_sales.ss_ext_sales_price)@3 as store_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[ca_county@0 as ca_county, d_qoy@1 as d_qoy, d_year@2 as d_year], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ca_county@0, d_qoy@1, d_year@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_county@3 as ca_county, d_qoy@2 as d_qoy, d_year@1 as d_year], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@0, CAST(customer_address.ca_address_sk AS Float64)@2)], projection=[ss_ext_sales_price@1, d_year@2, d_qoy@3, ca_county@5] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_addr_sk@2 as ss_addr_sk, ss_ext_sales_price@3 as ss_ext_sales_price, d_year@0 as d_year, d_qoy@1 as d_qoy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@0)], projection=[d_year@1, d_qoy@2, ss_addr_sk@5, ss_ext_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_county@1 as ca_county, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_county], file_type=parquet + │ ProjectionExec: expr=[ca_county@0 as ca_county, sum(web_sales.ws_ext_sales_price)@3 as web_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[ca_county@0 as ca_county, d_qoy@1 as d_qoy, d_year@2 as d_year], aggr=[sum(web_sales.ws_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ca_county@0, d_qoy@1, d_year@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_county@3 as ca_county, d_qoy@2 as d_qoy, d_year@1 as d_year], aggr=[sum(web_sales.ws_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_bill_addr_sk@0, CAST(customer_address.ca_address_sk AS Float64)@2)], projection=[ws_ext_sales_price@1, d_year@2, d_qoy@3, ca_county@5] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ws_bill_addr_sk@2 as ws_bill_addr_sk, ws_ext_sales_price@3 as ws_ext_sales_price, d_year@0 as d_year, d_qoy@1 as d_qoy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ws_sold_date_sk@0)], projection=[d_year@1, d_qoy@2, ws_bill_addr_sk@5, ws_ext_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_county@1 as ca_county, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_county], file_type=parquet + │ ProjectionExec: expr=[ca_county@0 as ca_county, sum(web_sales.ws_ext_sales_price)@3 as web_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[ca_county@0 as ca_county, d_qoy@1 as d_qoy, d_year@2 as d_year], aggr=[sum(web_sales.ws_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ca_county@0, d_qoy@1, d_year@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_county@3 as ca_county, d_qoy@2 as d_qoy, d_year@1 as d_year], aggr=[sum(web_sales.ws_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_bill_addr_sk@0, CAST(customer_address.ca_address_sk AS Float64)@2)], projection=[ws_ext_sales_price@1, d_year@2, d_qoy@3, ca_county@5] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ws_bill_addr_sk@2 as ws_bill_addr_sk, ws_ext_sales_price@3 as ws_ext_sales_price, d_year@0 as d_year, d_qoy@1 as d_qoy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ws_sold_date_sk@0)], projection=[d_year@1, d_qoy@2, ws_bill_addr_sk@5, ws_ext_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_county@1 as ca_county, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_county], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_qoy@2 = 1 AND d_year@1 = 2000 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@2 = 1 AND d_year@1 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 1 AND 1 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (1), d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_qoy@2 = 1 AND d_year@1 = 2000 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@2 = 1 AND d_year@1 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 1 AND 1 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (1), d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_qoy@2 = 2 AND d_year@1 = 2000 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@2 = 2 AND d_year@1 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_qoy@2 = 3 AND d_year@1 = 2000 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@2 = 3 AND d_year@1 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 3 AND 3 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (3), d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_qoy@2 = 2 AND d_year@1 = 2000 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@2 = 2 AND d_year@1 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_qoy@2 = 3 AND d_year@1 = 2000 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@2 = 3 AND d_year@1 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 3 AND 3 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (3), d_year in (2000)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_32() -> Result<()> { + let display = test_tpcds_query(32).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[sum(catalog_sales.cs_ext_discount_amt)@0 as excess discount amount] + │ GlobalLimitExec: skip=0, fetch=100 + │ AggregateExec: mode=Final, gby=[], aggr=[sum(catalog_sales.cs_ext_discount_amt)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(catalog_sales.cs_ext_discount_amt)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@1, i_item_sk@1)], filter=CAST(cs_ext_discount_amt@0 AS Decimal128(30, 15)) > Float64(1.3) * avg(catalog_sales.cs_ext_discount_amt)@1, projection=[cs_ext_discount_amt@2] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[CAST(1.3 * CAST(avg(catalog_sales.cs_ext_discount_amt)@1 AS Float64) AS Decimal128(30, 15)) as Float64(1.3) * avg(catalog_sales.cs_ext_discount_amt), cs_item_sk@0 as cs_item_sk] + │ AggregateExec: mode=FinalPartitioned, gby=[cs_item_sk@0 as cs_item_sk], aggr=[avg(catalog_sales.cs_ext_discount_amt)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cs_item_sk@0 as cs_item_sk], aggr=[avg(catalog_sales.cs_ext_discount_amt)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_item_sk@3, cs_ext_discount_amt@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_ext_discount_amt@3, i_item_sk@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_ext_discount_amt@2 as cs_ext_discount_amt, i_item_sk@0 as i_item_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_item_sk@0, cs_sold_date_sk@1, cs_ext_discount_amt@3] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-01-27 AND d_date@1 <= 2000-04-26, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-01-27 AND d_date@1 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-01-27 AND d_date@1 <= 2000-04-26, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-01-27 AND d_date@1 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_manufact_id@1 = 977, projection=[i_item_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=i_manufact_id@1 = 977 AND DynamicFilter [ empty ], pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 977 AND 977 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (977)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_33() -> Result<()> { + let display = test_tpcds_query(33).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [total_sales@1 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[total_sales@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(tmp1.total_sales)@1 as total_sales] + │ AggregateExec: mode=SinglePartitioned, gby=[i_manufact_id@0 as i_manufact_id], aggr=[sum(tmp1.total_sales)] + │ InterleaveExec + │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(store_sales.ss_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_manufact_id@0 as i_manufact_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_manufact_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_manufact_id@1 as i_manufact_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(i_manufact_id@1, i_manufact_id@0)] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_ext_sales_price@1, i_manufact_id@3] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@1)], projection=[ss_item_sk@0, ss_ext_sales_price@2] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_addr_sk@4, ss_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@1 = Some(-500),4,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_category@0 = Electronics, projection=[i_manufact_id@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@0 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] + │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(catalog_sales.cs_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_manufact_id@0 as i_manufact_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_manufact_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_manufact_id@1 as i_manufact_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(i_manufact_id@1, i_manufact_id@0)] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, i_item_sk@0)], projection=[cs_ext_sales_price@1, i_manufact_id@3] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_bill_addr_sk@0, CAST(customer_address.ca_address_sk AS Float64)@1)], projection=[cs_item_sk@1, cs_ext_sales_price@2] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_bill_addr_sk@3, cs_item_sk@4, cs_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@1 = Some(-500),4,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_category@0 = Electronics, projection=[i_manufact_id@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@0 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] + │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(web_sales.ws_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_manufact_id@0 as i_manufact_id], aggr=[sum(web_sales.ws_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_manufact_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_manufact_id@1 as i_manufact_id], aggr=[sum(web_sales.ws_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(i_manufact_id@1, i_manufact_id@0)] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@0, i_item_sk@0)], projection=[ws_ext_sales_price@1, i_manufact_id@3] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_bill_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@1)], projection=[ws_item_sk@0, ws_ext_sales_price@2] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_bill_addr_sk@4, ws_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@1 = Some(-500),4,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_category@0 = Electronics, projection=[i_manufact_id@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@0 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 5, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 1998 AND d_moy@2 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 5, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 1998 AND d_moy@2 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 5, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 1998 AND d_moy@2 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_34() -> Result<()> { + let display = test_tpcds_query(34).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c_last_name@0 ASC, c_first_name@1 ASC, c_salutation@2 ASC, c_preferred_cust_flag@3 DESC, ss_ticket_number@4 ASC] + │ SortExec: expr=[c_last_name@0 ASC, c_first_name@1 ASC, c_salutation@2 ASC, c_preferred_cust_flag@3 DESC, ss_ticket_number@4 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[c_last_name@4 as c_last_name, c_first_name@3 as c_first_name, c_salutation@2 as c_salutation, c_preferred_cust_flag@5 as c_preferred_cust_flag, ss_ticket_number@0 as ss_ticket_number, cnt@1 as cnt] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@1, CAST(customer.c_customer_sk AS Float64)@5)], projection=[ss_ticket_number@0, cnt@2, c_salutation@4, c_first_name@5, c_last_name@6, c_preferred_cust_flag@7] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, count(Int64(1))@2 as cnt] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: count(Int64(1))@2 >= 15 AND count(Int64(1))@2 <= 20 + │ AggregateExec: mode=FinalPartitioned, gby=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_ticket_number@0, ss_customer_sk@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_ticket_number@1 as ss_ticket_number, ss_customer_sk@0 as ss_customer_sk], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_customer_sk@2, ss_ticket_number@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@2)], projection=[ss_customer_sk@2, ss_hdemo_sk@3, ss_ticket_number@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_customer_sk@3, ss_hdemo_sk@4, ss_store_sk@5, ss_ticket_number@6] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_salutation@1 as c_salutation, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_salutation, c_first_name, c_last_name, c_preferred_cust_flag], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: (hd_buy_potential@1 = >10000 OR hd_buy_potential@1 = Unknown) AND hd_vehicle_count@3 > 0 AND CASE WHEN hd_vehicle_count@3 > 0 THEN CAST(hd_dep_count@2 AS Float64) / CAST(hd_vehicle_count@3 AS Float64) END > 1.2, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_buy_potential, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=(hd_buy_potential@1 = >10000 OR hd_buy_potential@1 = Unknown) AND hd_vehicle_count@3 > 0 AND CASE WHEN hd_vehicle_count@3 > 0 THEN CAST(hd_dep_count@2 AS Float64) / CAST(hd_vehicle_count@3 AS Float64) END > 1.2, pruning_predicate=(hd_buy_potential_null_count@2 != row_count@3 AND hd_buy_potential_min@0 <= >10000 AND >10000 <= hd_buy_potential_max@1 OR hd_buy_potential_null_count@2 != row_count@3 AND hd_buy_potential_min@0 <= Unknown AND Unknown <= hd_buy_potential_max@1) AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_max@4 > 0, required_guarantees=[hd_buy_potential in (>10000, Unknown)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_county@1 = Williamson County, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_county], file_type=parquet, predicate=s_county@1 = Williamson County, pruning_predicate=s_county_null_count@2 != row_count@3 AND s_county_min@0 <= Williamson County AND Williamson County <= s_county_max@1, required_guarantees=[s_county in (Williamson County)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: (d_dom@2 >= 1 AND d_dom@2 <= 3 OR d_dom@2 >= 25 AND d_dom@2 <= 28) AND (d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001), projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=(d_dom@2 >= 1 AND d_dom@2 <= 3 OR d_dom@2 >= 25 AND d_dom@2 <= 28) AND (d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001), pruning_predicate=(d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 3 OR d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 25 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 28) AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_35() -> Result<()> { + let display = test_tpcds_query(35).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [ca_state@0 ASC, cd_gender@1 ASC, cd_marital_status@2 ASC, cd_dep_count@3 ASC, cd_dep_employed_count@8 ASC, cd_dep_college_count@13 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[ca_state@0 ASC, cd_gender@1 ASC, cd_marital_status@2 ASC, cd_dep_count@3 ASC, cd_dep_employed_count@8 ASC, cd_dep_college_count@13 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[ca_state@0 as ca_state, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_dep_count@3 as cd_dep_count, count(Int64(1))@6 as cnt1, min(customer_demographics.cd_dep_count)@7 as min1, max(customer_demographics.cd_dep_count)@8 as max1, avg(customer_demographics.cd_dep_count)@9 as avg1, cd_dep_employed_count@4 as cd_dep_employed_count, count(Int64(1))@6 as cnt2, min(customer_demographics.cd_dep_employed_count)@10 as min2, max(customer_demographics.cd_dep_employed_count)@11 as max2, avg(customer_demographics.cd_dep_employed_count)@12 as avg2, cd_dep_college_count@5 as cd_dep_college_count, count(Int64(1))@6 as cnt3, min(customer_demographics.cd_dep_college_count)@13 as min(customer_demographics.cd_dep_college_count), max(customer_demographics.cd_dep_college_count)@14 as max(customer_demographics.cd_dep_college_count), avg(customer_demographics.cd_dep_college_count)@15 as avg(customer_demographics.cd_dep_college_count)] + │ AggregateExec: mode=FinalPartitioned, gby=[ca_state@0 as ca_state, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_dep_count@3 as cd_dep_count, cd_dep_employed_count@4 as cd_dep_employed_count, cd_dep_college_count@5 as cd_dep_college_count], aggr=[count(Int64(1)), min(customer_demographics.cd_dep_count), max(customer_demographics.cd_dep_count), avg(customer_demographics.cd_dep_count), min(customer_demographics.cd_dep_employed_count), max(customer_demographics.cd_dep_employed_count), avg(customer_demographics.cd_dep_employed_count), min(customer_demographics.cd_dep_college_count), max(customer_demographics.cd_dep_college_count), avg(customer_demographics.cd_dep_college_count)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ca_state@0, cd_gender@1, cd_marital_status@2, cd_dep_count@3, cd_dep_employed_count@4, cd_dep_college_count@5], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_state@0 as ca_state, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_dep_count@3 as cd_dep_count, cd_dep_employed_count@4 as cd_dep_employed_count, cd_dep_college_count@5 as cd_dep_college_count], aggr=[count(Int64(1)), min(customer_demographics.cd_dep_count), max(customer_demographics.cd_dep_count), avg(customer_demographics.cd_dep_count), min(customer_demographics.cd_dep_employed_count), max(customer_demographics.cd_dep_employed_count), avg(customer_demographics.cd_dep_employed_count), min(customer_demographics.cd_dep_college_count), max(customer_demographics.cd_dep_college_count), avg(customer_demographics.cd_dep_college_count)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: mark@6 OR mark@7, projection=[ca_state@0, cd_gender@1, cd_marital_status@2, cd_dep_count@3, cd_dep_employed_count@4, cd_dep_college_count@5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightMark, on=[(cs_ship_customer_sk@0, CAST(c.c_customer_sk AS Float64)@8)], projection=[ca_state@1, cd_gender@2, cd_marital_status@3, cd_dep_count@4, cd_dep_employed_count@5, cd_dep_college_count@6, mark@7, mark@9] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_ship_customer_sk@3] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, ca_state@1 as ca_state, cd_gender@2 as cd_gender, cd_marital_status@3 as cd_marital_status, cd_dep_count@4 as cd_dep_count, cd_dep_employed_count@5 as cd_dep_employed_count, cd_dep_college_count@6 as cd_dep_college_count, mark@7 as mark, CAST(c_customer_sk@0 AS Float64) as CAST(c.c_customer_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightMark, on=[(ws_bill_customer_sk@0, CAST(c.c_customer_sk AS Float64)@7)], projection=[c_customer_sk@0, ca_state@1, cd_gender@2, cd_marital_status@3, cd_dep_count@4, cd_dep_employed_count@5, cd_dep_college_count@6, mark@8] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_bill_customer_sk@3] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, ca_state@1 as ca_state, cd_gender@2 as cd_gender, cd_marital_status@3 as cd_marital_status, cd_dep_count@4 as cd_dep_count, cd_dep_employed_count@5 as cd_dep_employed_count, cd_dep_college_count@6 as cd_dep_college_count, CAST(c_customer_sk@0 AS Float64) as CAST(c.c_customer_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(ss_customer_sk@0, CAST(c.c_customer_sk AS Float64)@7)], projection=[c_customer_sk@0, ca_state@1, cd_gender@2, cd_marital_status@3, cd_dep_count@4, cd_dep_employed_count@5, cd_dep_college_count@6] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_customer_sk@3] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, ca_state@1 as ca_state, cd_gender@2 as cd_gender, cd_marital_status@3 as cd_marital_status, cd_dep_count@4 as cd_dep_count, cd_dep_employed_count@5 as cd_dep_employed_count, cd_dep_college_count@6 as cd_dep_college_count, CAST(c_customer_sk@0 AS Float64) as CAST(c.c_customer_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@1, CAST(customer_demographics.cd_demo_sk AS Float64)@6)], projection=[c_customer_sk@0, ca_state@2, cd_gender@4, cd_marital_status@5, cd_dep_count@6, cd_dep_employed_count@7, cd_dep_college_count@8] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_dep_count@3 as cd_dep_count, cd_dep_employed_count@4 as cd_dep_employed_count, cd_dep_college_count@5 as cd_dep_college_count, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002 AND d_qoy@2 < 4, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@1 = 2002 AND d_qoy@2 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002 AND d_qoy@2 < 4, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@1 = 2002 AND d_qoy@2 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002 AND d_qoy@2 < 4, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@1 = 2002 AND d_qoy@2 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ ProjectionExec: expr=[c_customer_sk@1 as c_customer_sk, c_current_cdemo_sk@2 as c_current_cdemo_sk, ca_state@0 as ca_state] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@2)], projection=[ca_state@1, c_customer_sk@2, c_current_cdemo_sk@3] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ca_address_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_current_addr_sk@2], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_36() -> Result<()> { + let display = test_tpcds_query(36).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [lochierarchy@3 DESC, CASE WHEN lochierarchy@3 = 0 THEN i_category@1 END ASC, rank_within_parent@4 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[lochierarchy@3 DESC, CASE WHEN lochierarchy@3 = 0 THEN i_category@1 END ASC, rank_within_parent@4 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[gross_margin@0 as gross_margin, i_category@1 as i_category, i_class@2 as i_class, lochierarchy@4 as lochierarchy, rank() PARTITION BY [results_rollup.lochierarchy, CASE WHEN results_rollup.t_class = Int64(0) THEN results_rollup.i_category END] ORDER BY [results_rollup.gross_margin ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@5 as rank_within_parent] + │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [results_rollup.lochierarchy, CASE WHEN results_rollup.t_class = Int64(0) THEN results_rollup.i_category END] ORDER BY [results_rollup.gross_margin ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [results_rollup.lochierarchy, CASE WHEN results_rollup.t_class = Int64(0) THEN results_rollup.i_category END] ORDER BY [results_rollup.gross_margin ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[lochierarchy@4 ASC NULLS LAST, CASE WHEN t_class@3 = 0 THEN i_category@1 END ASC NULLS LAST, gross_margin@0 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([lochierarchy@4, CASE WHEN t_class@3 = 0 THEN i_category@1 END], 3), input_partitions=3 + │ ProjectionExec: expr=[gross_margin@0 as gross_margin, i_category@1 as i_category, i_class@2 as i_class, t_class@4 as t_class, lochierarchy@5 as lochierarchy] + │ AggregateExec: mode=FinalPartitioned, gby=[gross_margin@0 as gross_margin, i_category@1 as i_category, i_class@2 as i_class, t_category@3 as t_category, t_class@4 as t_class, lochierarchy@5 as lochierarchy], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([gross_margin@0, i_category@1, i_class@2, t_category@3, t_class@4, lochierarchy@5], 3), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[gross_margin@0 as gross_margin, i_category@1 as i_category, i_class@2 as i_class, t_category@3 as t_category, t_class@4 as t_class, lochierarchy@5 as lochierarchy], aggr=[], ordering_mode=PartiallySorted([3]) + │ UnionExec + │ ProjectionExec: expr=[gross_margin@0 as gross_margin, CAST(i_category@1 AS Utf8) as i_category, i_class@2 as i_class, t_category@3 as t_category, t_class@4 as t_class, lochierarchy@5 as lochierarchy] + │ AggregateExec: mode=FinalPartitioned, gby=[gross_margin@0 as gross_margin, i_category@1 as i_category, i_class@2 as i_class, t_category@3 as t_category, t_class@4 as t_class, lochierarchy@5 as lochierarchy], aggr=[], ordering_mode=PartiallySorted([3]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([gross_margin@0, i_category@1, i_class@2, t_category@3, t_class@4, lochierarchy@5], 3), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[gross_margin@0 as gross_margin, i_category@1 as i_category, i_class@2 as i_class, t_category@3 as t_category, t_class@4 as t_class, lochierarchy@5 as lochierarchy], aggr=[], ordering_mode=PartiallySorted([3, 4, 5]) + │ UnionExec + │ ProjectionExec: expr=[gross_margin@0 as gross_margin, i_category@1 as i_category, CAST(i_class@2 AS Utf8) as i_class, 0 as t_category, 0 as t_class, 0 as lochierarchy] + │ ProjectionExec: expr=[CAST(sum(store_sales.ss_net_profit)@2 AS Float64) / CAST(sum(store_sales.ss_ext_sales_price)@3 AS Float64) as gross_margin, i_category@0 as i_category, i_class@1 as i_class] + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_class@1 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_class@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@3 as i_category, i_class@2 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)], projection=[ss_ext_sales_price@3, ss_net_profit@4, i_class@5, i_category@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_store_sk@1, ss_ext_sales_price@2, ss_net_profit@3, i_class@5, i_category@6] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_ext_sales_price@5, ss_net_profit@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[CAST(sum(results.ss_net_profit)@1 AS Float64) / CAST(sum(results.ss_ext_sales_price)@2 AS Float64) as gross_margin, i_category@0 as i_category, NULL as i_class, 0 as t_category, 1 as t_class, 1 as lochierarchy] + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category], aggr=[sum(results.ss_net_profit), sum(results.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@2 as i_category], aggr=[sum(results.ss_net_profit), sum(results.ss_ext_sales_price)] + │ ProjectionExec: expr=[sum(store_sales.ss_net_profit)@2 as ss_net_profit, sum(store_sales.ss_ext_sales_price)@3 as ss_ext_sales_price, i_category@0 as i_category] + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_class@1 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_class@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@3 as i_category, i_class@2 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)], projection=[ss_ext_sales_price@3, ss_net_profit@4, i_class@5, i_category@6] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_store_sk@1, ss_ext_sales_price@2, ss_net_profit@3, i_class@5, i_category@6] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_ext_sales_price@5, ss_net_profit@6] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[CAST(sum(results.ss_net_profit)@0 AS Float64) / CAST(sum(results.ss_ext_sales_price)@1 AS Float64) as gross_margin, NULL as i_category, NULL as i_class, 1 as t_category, 1 as t_class, 2 as lochierarchy] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(results.ss_net_profit), sum(results.ss_ext_sales_price)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(results.ss_net_profit), sum(results.ss_ext_sales_price)] + │ ProjectionExec: expr=[sum(store_sales.ss_net_profit)@2 as ss_net_profit, sum(store_sales.ss_ext_sales_price)@3 as ss_ext_sales_price] + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_class@1 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_class@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@3 as i_category, i_class@2 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)], projection=[ss_ext_sales_price@3, ss_net_profit@4, i_class@5, i_category@6] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_store_sk@1, ss_ext_sales_price@2, ss_net_profit@3, i_class@5, i_category@6] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_ext_sales_price@5, ss_net_profit@6] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@1 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@1 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@1 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_37() -> Result<()> { + let display = test_tpcds_query(37).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, i_current_price@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@0)], projection=[i_item_id@1, i_item_desc@2, i_current_price@3] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(inv_date_sk@4, d_date_sk@0)], projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[i_item_sk@1 as i_item_sk, i_item_id@2 as i_item_id, i_item_desc@3 as i_item_desc, i_current_price@4 as i_current_price, inv_date_sk@0 as inv_date_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(inv_item_sk@1, i_item_sk@0)], projection=[inv_date_sk@0, i_item_sk@2, i_item_id@3, i_item_desc@4, i_current_price@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_current_price@3 >= Some(6800),4,2 AND i_current_price@3 <= Some(9800),4,2 AND i_manufact_id@4 IN (SET) ([677, 940, 694, 808]), projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@3 >= Some(6800),4,2 AND i_current_price@3 <= Some(9800),4,2 AND i_manufact_id@4 IN (SET) ([677, 940, 694, 808]) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(6800),4,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(9800),4,2 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 677 AND 677 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 940 AND 940 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 694 AND 694 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 808 AND 808 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (677, 694, 808, 940)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-02-01 AND d_date@1 <= 2000-04-01, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-02-01 AND d_date@1 <= 2000-04-01 AND DynamicFilter [ empty ], pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-01, required_guarantees=[] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: inv_quantity_on_hand@2 >= 100 AND inv_quantity_on_hand@2 <= 500, projection=[inv_date_sk@0, inv_item_sk@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], ...]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@2 >= 100 AND inv_quantity_on_hand@2 <= 500, pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_38() -> Result<()> { + let display = test_tpcds_query(38).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ GlobalLimitExec: skip=0, fetch=100 + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(c_last_name@0, c_last_name@0), (c_first_name@1, c_first_name@1), (d_date@2, d_date@2)], NullsEqual: true + │ CoalescePartitionsExec + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, d_date@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ ProjectionExec: expr=[c_last_name@2 as c_last_name, c_first_name@1 as c_first_name, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_bill_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@3)], projection=[d_date@1, c_first_name@3, c_last_name@4] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ws_bill_customer_sk@1 as ws_bill_customer_sk, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@0)], projection=[d_date@1, ws_bill_customer_sk@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ AggregateExec: mode=SinglePartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(c_last_name@0, c_last_name@0), (c_first_name@1, c_first_name@1), (d_date@2, d_date@2)], NullsEqual: true + │ CoalescePartitionsExec + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, d_date@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ ProjectionExec: expr=[c_last_name@2 as c_last_name, c_first_name@1 as c_first_name, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_bill_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@3)], projection=[d_date@1, c_first_name@3, c_last_name@4] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[cs_bill_customer_sk@1 as cs_bill_customer_sk, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@0)], projection=[d_date@1, cs_bill_customer_sk@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, d_date@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ ProjectionExec: expr=[c_last_name@2 as c_last_name, c_first_name@1 as c_first_name, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@3)], projection=[d_date@1, c_first_name@3, c_last_name@4] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_customer_sk@1 as ss_customer_sk, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_date@1, ss_customer_sk@4] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_39() -> Result<()> { + let display = test_tpcds_query(39).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[wsk1@0 as wsk1, isk1@1 as isk1, dmoy1@2 as dmoy1, mean1@3 as mean1, cov1@4 as cov1, w_warehouse_sk@5 as w_warehouse_sk, i_item_sk@6 as i_item_sk, d_moy@7 as d_moy, mean@8 as mean, cov@9 as cov] + │ SortPreservingMergeExec: [w_warehouse_sk@10 ASC, i_item_sk@11 ASC, d_moy@12 ASC, mean@13 ASC, cov@14 ASC, d_moy@7 ASC, mean@8 ASC, cov@9 ASC] + │ SortExec: expr=[wsk1@0 ASC, isk1@1 ASC, mean1@3 ASC, cov1@4 ASC, mean@8 ASC, cov@9 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[w_warehouse_sk@0 as wsk1, i_item_sk@1 as isk1, d_moy@2 as dmoy1, mean@3 as mean1, cov@4 as cov1, w_warehouse_sk@5 as w_warehouse_sk, i_item_sk@6 as i_item_sk, d_moy@7 as d_moy, mean@8 as mean, cov@9 as cov, w_warehouse_sk@0 as w_warehouse_sk, i_item_sk@1 as i_item_sk, d_moy@2 as d_moy, mean@3 as mean, cov@4 as cov] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@1, i_item_sk@1), (w_warehouse_sk@0, w_warehouse_sk@0)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@1, w_warehouse_sk@0], 3), input_partitions=3 + │ ProjectionExec: expr=[w_warehouse_sk@0 as w_warehouse_sk, i_item_sk@1 as i_item_sk, d_moy@2 as d_moy, avg(inventory.inv_quantity_on_hand)@4 as mean, CASE avg(inventory.inv_quantity_on_hand)@4 WHEN 0 THEN NULL ELSE stddev(inventory.inv_quantity_on_hand)@3 / avg(inventory.inv_quantity_on_hand)@4 END as cov] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CASE avg(inventory.inv_quantity_on_hand)@4 WHEN 0 THEN 0 ELSE stddev(inventory.inv_quantity_on_hand)@3 / avg(inventory.inv_quantity_on_hand)@4 END > 1 + │ ProjectionExec: expr=[w_warehouse_sk@1 as w_warehouse_sk, i_item_sk@2 as i_item_sk, d_moy@3 as d_moy, stddev(inventory.inv_quantity_on_hand)@4 as stddev(inventory.inv_quantity_on_hand), avg(inventory.inv_quantity_on_hand)@5 as avg(inventory.inv_quantity_on_hand)] + │ AggregateExec: mode=FinalPartitioned, gby=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sk@1 as w_warehouse_sk, i_item_sk@2 as i_item_sk, d_moy@3 as d_moy], aggr=[stddev(inventory.inv_quantity_on_hand), avg(inventory.inv_quantity_on_hand)], ordering_mode=PartiallySorted([3]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([w_warehouse_name@0, w_warehouse_sk@1, i_item_sk@2, d_moy@3], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[w_warehouse_name@3 as w_warehouse_name, w_warehouse_sk@2 as w_warehouse_sk, i_item_sk@1 as i_item_sk, d_moy@4 as d_moy], aggr=[stddev(inventory.inv_quantity_on_hand), avg(inventory.inv_quantity_on_hand)], ordering_mode=PartiallySorted([3]) + │ ProjectionExec: expr=[inv_quantity_on_hand@1 as inv_quantity_on_hand, i_item_sk@2 as i_item_sk, w_warehouse_sk@3 as w_warehouse_sk, w_warehouse_name@4 as w_warehouse_name, d_moy@0 as d_moy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, inv_date_sk@0)], projection=[d_moy@1, inv_quantity_on_hand@3, i_item_sk@4, w_warehouse_sk@5, w_warehouse_name@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[inv_date_sk@2 as inv_date_sk, inv_quantity_on_hand@3 as inv_quantity_on_hand, i_item_sk@4 as i_item_sk, w_warehouse_sk@0 as w_warehouse_sk, w_warehouse_name@1 as w_warehouse_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(w_warehouse_sk@0, inv_warehouse_sk@1)], projection=[w_warehouse_sk@0, w_warehouse_name@1, inv_date_sk@2, inv_quantity_on_hand@4, i_item_sk@5] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/warehouse/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet]]}, projection=[w_warehouse_sk, w_warehouse_name], file_type=parquet + │ ProjectionExec: expr=[inv_date_sk@1 as inv_date_sk, inv_warehouse_sk@2 as inv_warehouse_sk, inv_quantity_on_hand@3 as inv_quantity_on_hand, i_item_sk@0 as i_item_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, inv_item_sk@1)], projection=[i_item_sk@0, inv_date_sk@1, inv_warehouse_sk@3, inv_quantity_on_hand@4] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@1, w_warehouse_sk@0], 3), input_partitions=3 + │ ProjectionExec: expr=[w_warehouse_sk@0 as w_warehouse_sk, i_item_sk@1 as i_item_sk, d_moy@2 as d_moy, avg(inventory.inv_quantity_on_hand)@4 as mean, CASE avg(inventory.inv_quantity_on_hand)@4 WHEN 0 THEN NULL ELSE stddev(inventory.inv_quantity_on_hand)@3 / avg(inventory.inv_quantity_on_hand)@4 END as cov] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CASE avg(inventory.inv_quantity_on_hand)@4 WHEN 0 THEN 0 ELSE stddev(inventory.inv_quantity_on_hand)@3 / avg(inventory.inv_quantity_on_hand)@4 END > 1 + │ ProjectionExec: expr=[w_warehouse_sk@1 as w_warehouse_sk, i_item_sk@2 as i_item_sk, d_moy@3 as d_moy, stddev(inventory.inv_quantity_on_hand)@4 as stddev(inventory.inv_quantity_on_hand), avg(inventory.inv_quantity_on_hand)@5 as avg(inventory.inv_quantity_on_hand)] + │ AggregateExec: mode=FinalPartitioned, gby=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sk@1 as w_warehouse_sk, i_item_sk@2 as i_item_sk, d_moy@3 as d_moy], aggr=[stddev(inventory.inv_quantity_on_hand), avg(inventory.inv_quantity_on_hand)], ordering_mode=PartiallySorted([3]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([w_warehouse_name@0, w_warehouse_sk@1, i_item_sk@2, d_moy@3], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[w_warehouse_name@3 as w_warehouse_name, w_warehouse_sk@2 as w_warehouse_sk, i_item_sk@1 as i_item_sk, d_moy@4 as d_moy], aggr=[stddev(inventory.inv_quantity_on_hand), avg(inventory.inv_quantity_on_hand)], ordering_mode=PartiallySorted([3]) + │ ProjectionExec: expr=[inv_quantity_on_hand@1 as inv_quantity_on_hand, i_item_sk@2 as i_item_sk, w_warehouse_sk@3 as w_warehouse_sk, w_warehouse_name@4 as w_warehouse_name, d_moy@0 as d_moy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, inv_date_sk@0)], projection=[d_moy@1, inv_quantity_on_hand@3, i_item_sk@4, w_warehouse_sk@5, w_warehouse_name@6] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[inv_date_sk@2 as inv_date_sk, inv_quantity_on_hand@3 as inv_quantity_on_hand, i_item_sk@4 as i_item_sk, w_warehouse_sk@0 as w_warehouse_sk, w_warehouse_name@1 as w_warehouse_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(w_warehouse_sk@0, inv_warehouse_sk@1)], projection=[w_warehouse_sk@0, w_warehouse_name@1, inv_date_sk@2, inv_quantity_on_hand@4, i_item_sk@5] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/warehouse/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet]]}, projection=[w_warehouse_sk, w_warehouse_name], file_type=parquet + │ ProjectionExec: expr=[inv_date_sk@1 as inv_date_sk, inv_warehouse_sk@2 as inv_warehouse_sk, inv_quantity_on_hand@3 as inv_quantity_on_hand, i_item_sk@0 as i_item_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, inv_item_sk@1)], projection=[i_item_sk@0, inv_date_sk@1, inv_warehouse_sk@3, inv_quantity_on_hand@4] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_moy@2 = 1 AND d_year@1 = 2001, projection=[d_date_sk@0, d_moy@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 1 AND d_year@1 = 2001, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 1 AND 1 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_moy in (1), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([inv_item_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], ...]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_moy@2 = 2 AND d_year@1 = 2001, projection=[d_date_sk@0, d_moy@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 2 AND d_year@1 = 2001, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 2 AND 2 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([inv_item_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], ...]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_40() -> Result<()> { + let display = test_tpcds_query(40).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [w_state@0 ASC NULLS LAST, i_item_id@1 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[w_state@0 ASC NULLS LAST, i_item_id@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[w_state@0 as w_state, i_item_id@1 as i_item_id, sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END)@2 as sales_before, sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END)@3 as sales_after] + │ AggregateExec: mode=FinalPartitioned, gby=[w_state@0 as w_state, i_item_id@1 as i_item_id], aggr=[sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END), sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([w_state@0, i_item_id@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[w_state@3 as w_state, i_item_id@4 as i_item_id], aggr=[sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END), sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END)] + │ ProjectionExec: expr=[d_date@4 as __common_expr_1, cs_sales_price@0 as cs_sales_price, cr_refunded_cash@1 as cr_refunded_cash, w_state@2 as w_state, i_item_id@3 as i_item_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_sold_date_sk@0, CAST(date_dim.d_date_sk AS Float64)@2)], projection=[cs_sales_price@1, cr_refunded_cash@2, w_state@3, i_item_id@4, d_date@6] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@1, i_item_sk@0)], projection=[cs_sold_date_sk@0, cs_sales_price@2, cr_refunded_cash@3, w_state@4, i_item_id@6] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_item_sk@2 as cs_item_sk, cs_sales_price@3 as cs_sales_price, cr_refunded_cash@4 as cr_refunded_cash, w_state@0 as w_state] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(warehouse.w_warehouse_sk AS Float64)@2, cs_warehouse_sk@1)], projection=[w_state@1, cs_sold_date_sk@3, cs_item_sk@5, cs_sales_price@6, cr_refunded_cash@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_warehouse_sk@2 as cs_warehouse_sk, cs_item_sk@3 as cs_item_sk, cs_sales_price@4 as cs_sales_price, cr_refunded_cash@0 as cr_refunded_cash] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(cr_order_number@1, cs_order_number@3), (cr_item_sk@0, cs_item_sk@2)], projection=[cr_refunded_cash@2, cs_sold_date_sk@3, cs_warehouse_sk@4, cs_item_sk@5, cs_sales_price@7] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_current_price@2 >= Some(99),4,2 AND i_current_price@2 <= Some(149),4,2, projection=[i_item_sk@0, i_item_id@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_current_price], file_type=parquet, predicate=i_current_price@2 >= Some(99),4,2 AND i_current_price@2 <= Some(149),4,2 AND DynamicFilter [ empty ], pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(99),4,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(149),4,2, required_guarantees=[] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-02-10 AND d_date@1 <= 2000-04-10 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-02-10 AND d_date@1 <= 2000-04-10, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-10 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-10, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[w_warehouse_sk@0 as w_warehouse_sk, w_state@1 as w_state, CAST(w_warehouse_sk@0 AS Float64) as CAST(warehouse.w_warehouse_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet]]}, projection=[w_warehouse_sk, w_state], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cr_order_number@1, cr_item_sk@0], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_refunded_cash], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_order_number@3, cs_item_sk@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_warehouse_sk, cs_item_sk, cs_order_number, cs_sales_price], file_type=parquet + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_41() -> Result<()> { + let display = test_tpcds_query(41).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_product_name@0 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_product_name@0 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[i_product_name@0 as i_product_name], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_product_name@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_product_name@0 as i_product_name], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CASE WHEN __always_true@2 IS NULL THEN 0 ELSE item_cnt@1 END > 0, projection=[i_product_name@0] + │ ProjectionExec: expr=[i_product_name@2 as i_product_name, item_cnt@0 as item_cnt, __always_true@1 as __always_true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(i_manufact@1, i_manufact@0)], projection=[item_cnt@0, __always_true@2, i_product_name@4] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[count(Int64(1))@1 as item_cnt, i_manufact@0 as i_manufact, true as __always_true] + │ AggregateExec: mode=FinalPartitioned, gby=[i_manufact@0 as i_manufact], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_manufact_id@0 >= 738 AND i_manufact_id@0 <= 778, projection=[i_manufact@1, i_product_name@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_manufact_id, i_manufact, i_product_name], file_type=parquet, predicate=i_manufact_id@0 >= 738 AND i_manufact_id@0 <= 778, pruning_predicate=i_manufact_id_null_count@1 != row_count@2 AND i_manufact_id_max@0 >= 738 AND i_manufact_id_null_count@1 != row_count@2 AND i_manufact_id_min@3 <= 778, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_manufact@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_manufact@0 as i_manufact], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: __common_expr_4@0 AND ((i_color@4 = powder OR i_color@4 = khaki) AND (i_units@5 = Ounce OR i_units@5 = Oz) AND (i_size@3 = medium OR i_size@3 = extra large) OR (i_color@4 = brown OR i_color@4 = honeydew) AND (i_units@5 = Bunch OR i_units@5 = Ton) AND (i_size@3 = N/A OR i_size@3 = small)) OR i_category@1 = Men AND (i_color@4 = floral OR i_color@4 = deep) AND (i_units@5 = N/A OR i_units@5 = Dozen) AND i_size@3 = petite OR i_category@1 = Men AND (i_color@4 = light OR i_color@4 = cornflower) AND (i_units@5 = Box OR i_units@5 = Pound) AND (i_size@3 = medium OR i_size@3 = extra large) OR __common_expr_4@0 AND ((i_color@4 = midnight OR i_color@4 = snow) AND (i_units@5 = Pallet OR i_units@5 = Gross) AND (i_size@3 = medium OR i_size@3 = extra large) OR (i_color@4 = cyan OR i_color@4 = papaya) AND (i_units@5 = Cup OR i_units@5 = Dram) AND (i_size@3 = N/A OR i_size@3 = small)) OR i_category@1 = Men AND (i_color@4 = orange OR i_color@4 = frosted) AND (i_units@5 = Each OR i_units@5 = Tbl) AND i_size@3 = petite OR i_category@1 = Men AND (i_color@4 = forest OR i_color@4 = ghost) AND (i_units@5 = Lb OR i_units@5 = Bundle) AND (i_size@3 = medium OR i_size@3 = extra large), projection=[i_manufact@2] + │ ProjectionExec: expr=[i_category@0 = Women as __common_expr_4, i_category@0 as i_category, i_manufact@1 as i_manufact, i_size@2 as i_size, i_color@3 as i_color, i_units@4 as i_units] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_category, i_manufact, i_size, i_color, i_units], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_42() -> Result<()> { + let display = test_tpcds_query(42).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [sum(store_sales.ss_ext_sales_price)@3 DESC, d_year@0 ASC NULLS LAST, i_category_id@1 ASC NULLS LAST, i_category@2 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[sum(store_sales.ss_ext_sales_price)@3 DESC, i_category_id@1 ASC NULLS LAST, i_category@2 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_category_id@1 as i_category_id, i_category@2 as i_category], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([0]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([d_year@0, i_category_id@1, i_category@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_category_id@2 as i_category_id, i_category@3 as i_category], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([0]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@1, i_item_sk@0)], projection=[d_year@0, ss_ext_sales_price@2, i_category_id@4, i_category@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(dt.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_manager_id@3 = 1, projection=[i_item_sk@0, i_category_id@1, i_category@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category_id, i_category, i_manager_id], file_type=parquet, predicate=i_manager_id@3 = 1 AND DynamicFilter [ empty ], pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(dt.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_moy@2 = 11 AND d_year@1 = 2000, projection=[d_date_sk@0, d_year@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 11 AND d_year@1 = 2000, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (2000)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_43() -> Result<()> { + let display = test_tpcds_query(43).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [s_store_name@0 ASC NULLS LAST, s_store_id@1 ASC NULLS LAST, sun_sales@2 ASC NULLS LAST, mon_sales@3 ASC NULLS LAST, tue_sales@4 ASC NULLS LAST, wed_sales@5 ASC NULLS LAST, thu_sales@6 ASC NULLS LAST, fri_sales@7 ASC NULLS LAST, sat_sales@8 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[s_store_name@0 ASC NULLS LAST, s_store_id@1 ASC NULLS LAST, sun_sales@2 ASC NULLS LAST, mon_sales@3 ASC NULLS LAST, tue_sales@4 ASC NULLS LAST, wed_sales@5 ASC NULLS LAST, thu_sales@6 ASC NULLS LAST, fri_sales@7 ASC NULLS LAST, sat_sales@8 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[s_store_name@0 as s_store_name, s_store_id@1 as s_store_id, sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END)@2 as sun_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END)@3 as mon_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END)@4 as tue_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END)@5 as wed_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END)@6 as thu_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END)@7 as fri_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)@8 as sat_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_name@0 as s_store_name, s_store_id@1 as s_store_id], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([s_store_name@0, s_store_id@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[s_store_name@3 as s_store_name, s_store_id@2 as s_store_id], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] + │ ProjectionExec: expr=[d_day_name@2 as d_day_name, ss_sales_price@3 as ss_sales_price, s_store_id@0 as s_store_id, s_store_name@1 as s_store_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@3, ss_store_sk@1)], projection=[s_store_id@1, s_store_name@2, d_day_name@4, ss_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_day_name@1, ss_store_sk@4, ss_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_id@1 as s_store_id, s_store_name@2 as s_store_name, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_gmt_offset@3 = Some(-500),3,2, projection=[s_store_sk@0, s_store_id@1, s_store_name@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id, s_store_name, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@3 = Some(-500),3,2, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= Some(-500),3,2 AND Some(-500),3,2 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (Some(-500),3,2)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_day_name@1 as d_day_name, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0, d_day_name@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_day_name], file_type=parquet, predicate=d_year@1 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_44() -> Result<()> { + let display = test_tpcds_query(44).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [rnk@0 ASC NULLS LAST], fetch=100 + │ [Stage 9] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[rnk@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[rnk@0 as rnk, i_product_name@1 as best_performing, i_product_name@2 as worst_performing] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(item_sk@1, i_item_sk@0)], projection=[rnk@0, i_product_name@2, i_product_name@4] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([item_sk@1], 6), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(item_sk@0, i_item_sk@0)], projection=[rnk@1, item_sk@2, i_product_name@4] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=1 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([item_sk@0], 6), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(rnk@1, rnk@1)], projection=[item_sk@0, rnk@1, item_sk@2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([rnk@1], 3), input_partitions=3 + │ ProjectionExec: expr=[item_sk@0 as item_sk, rank() ORDER BY [v1.rank_col ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 as rnk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: rank() ORDER BY [v1.rank_col ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 < 11 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1 + │ ProjectionExec: expr=[item_sk@0 as item_sk, rank() ORDER BY [v1.rank_col ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as rank() ORDER BY [v1.rank_col ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [v1.rank_col ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [v1.rank_col ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortPreservingMergeExec: [rank_col@1 ASC NULLS LAST] + │ SortExec: expr=[rank_col@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[ss_item_sk@0 as item_sk, avg(ss1.ss_net_profit)@1 as rank_col] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CAST(avg(ss1.ss_net_profit)@1 AS Decimal128(30, 15)) > CAST(0.9 * rank_col@2 AS Decimal128(30, 15)), projection=[ss_item_sk@0, avg(ss1.ss_net_profit)@1] + │ NestedLoopJoinExec: join_type=Left + │ CoalescePartitionsExec + │ AggregateExec: mode=FinalPartitioned, gby=[ss_item_sk@0 as ss_item_sk], aggr=[avg(ss1.ss_net_profit)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[CAST(avg(store_sales.ss_net_profit)@1 AS Float64) as rank_col] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(store_sales.ss_net_profit)], ordering_mode=Sorted + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([rnk@1], 3), input_partitions=3 + │ ProjectionExec: expr=[item_sk@0 as item_sk, rank() ORDER BY [v2.rank_col DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 as rnk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: rank() ORDER BY [v2.rank_col DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 < 11 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1 + │ ProjectionExec: expr=[item_sk@0 as item_sk, rank() ORDER BY [v2.rank_col DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as rank() ORDER BY [v2.rank_col DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [v2.rank_col DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [v2.rank_col DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortPreservingMergeExec: [rank_col@1 DESC] + │ SortExec: expr=[rank_col@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[ss_item_sk@0 as item_sk, avg(ss1.ss_net_profit)@1 as rank_col] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CAST(avg(ss1.ss_net_profit)@1 AS Decimal128(30, 15)) > CAST(0.9 * rank_col@2 AS Decimal128(30, 15)), projection=[ss_item_sk@0, avg(ss1.ss_net_profit)@1] + │ NestedLoopJoinExec: join_type=Left + │ CoalescePartitionsExec + │ AggregateExec: mode=FinalPartitioned, gby=[ss_item_sk@0 as ss_item_sk], aggr=[avg(ss1.ss_net_profit)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[CAST(avg(store_sales.ss_net_profit)@1 AS Float64) as rank_col] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(store_sales.ss_net_profit)], ordering_mode=Sorted + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_item_sk@0 as ss_item_sk], aggr=[avg(ss1.ss_net_profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_store_sk@1 = 4, projection=[ss_item_sk@0, ss_net_profit@2] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_item_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=ss_store_sk@1 = 4, pruning_predicate=ss_store_sk_null_count@2 != row_count@3 AND ss_store_sk_min@0 <= 4 AND 4 <= ss_store_sk_max@1, required_guarantees=[ss_store_sk in (4)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_store_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(store_sales.ss_net_profit)], ordering_mode=Sorted + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_store_sk@1 = 4 AND ss_addr_sk@0 IS NULL, projection=[ss_store_sk@1, ss_net_profit@2] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_addr_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=ss_store_sk@1 = 4 AND ss_addr_sk@0 IS NULL, pruning_predicate=ss_store_sk_null_count@2 != row_count@3 AND ss_store_sk_min@0 <= 4 AND 4 <= ss_store_sk_max@1 AND ss_addr_sk_null_count@4 > 0, required_guarantees=[ss_store_sk in (4)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_item_sk@0 as ss_item_sk], aggr=[avg(ss1.ss_net_profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_store_sk@1 = 4, projection=[ss_item_sk@0, ss_net_profit@2] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_item_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=ss_store_sk@1 = 4, pruning_predicate=ss_store_sk_null_count@2 != row_count@3 AND ss_store_sk_min@0 <= 4 AND 4 <= ss_store_sk_max@1, required_guarantees=[ss_store_sk in (4)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_store_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(store_sales.ss_net_profit)], ordering_mode=Sorted + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_store_sk@1 = 4 AND ss_addr_sk@0 IS NULL, projection=[ss_store_sk@1, ss_net_profit@2] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_addr_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=ss_store_sk@1 = 4 AND ss_addr_sk@0 IS NULL, pruning_predicate=ss_store_sk_null_count@2 != row_count@3 AND ss_store_sk_min@0 <= 4 AND 4 <= ss_store_sk_max@1 AND ss_addr_sk_null_count@4 > 0, required_guarantees=[ss_store_sk in (4)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_product_name], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_product_name], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_45() -> Result<()> { + let display = test_tpcds_query(45).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [ca_zip@0 ASC NULLS LAST, ca_city@1 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[ca_zip@0 ASC NULLS LAST, ca_city@1 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[ca_zip@0 as ca_zip, ca_city@1 as ca_city], aggr=[sum(web_sales.ws_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ca_zip@0, ca_city@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_zip@2 as ca_zip, ca_city@1 as ca_city], aggr=[sum(web_sales.ws_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: substr(ca_zip@2, 1, 5) IN ([85669, 86197, 88274, 83405, 86475, 85392, 85460, 80348, 81792]) OR mark@3, projection=[ws_sales_price@0, ca_city@1, ca_zip@2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftMark, on=[(i_item_id@3, i_item_id@0)], projection=[ws_sales_price@0, ca_city@1, ca_zip@2, mark@4] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@0, i_item_sk@0)], projection=[ws_sales_price@1, ca_city@2, ca_zip@3, i_item_id@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_sales_price@4, ca_city@5, ca_zip@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@2 as ws_sold_date_sk, ws_item_sk@3 as ws_item_sk, ws_sales_price@4 as ws_sales_price, ca_city@0 as ca_city, ca_zip@1 as ca_zip] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@3)], projection=[ca_city@1, ca_zip@2, ws_sold_date_sk@3, ws_item_sk@4, ws_sales_price@5] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_item_sk@0 IN (SET) ([2, 3, 5, 7, 11, 13, 17, 19, 23, 29]), projection=[i_item_id@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=i_item_sk@0 IN (SET) ([2, 3, 5, 7, 11, 13, 17, 19, 23, 29]), pruning_predicate=i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 2 AND 2 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 3 AND 3 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 5 AND 5 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 7 AND 7 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 11 AND 11 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 13 AND 13 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 17 AND 17 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 19 AND 19 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 23 AND 23 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 29 AND 29 <= i_item_sk_max@1, required_guarantees=[i_item_sk in (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_qoy@2 = 2 AND d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@2 = 2 AND d_year@1 = 2001, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ca_address_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_city, ca_zip], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_current_addr_sk@3], 3), input_partitions=3 + │ ProjectionExec: expr=[ws_sold_date_sk@1 as ws_sold_date_sk, ws_item_sk@2 as ws_item_sk, ws_sales_price@3 as ws_sales_price, c_current_addr_sk@0 as c_current_addr_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@2, ws_bill_customer_sk@2)], projection=[c_current_addr_sk@1, ws_sold_date_sk@3, ws_item_sk@4, ws_sales_price@6] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@2], 6), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@2], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_46() -> Result<()> { + let display = test_tpcds_query(46).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c_last_name@0 ASC, c_first_name@1 ASC, ca_city@2 ASC, bought_city@3 ASC, ss_ticket_number@4 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[c_last_name@0 ASC, c_first_name@1 ASC, ca_city@2 ASC, bought_city@3 ASC, ss_ticket_number@4 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[c_last_name@5 as c_last_name, c_first_name@4 as c_first_name, ca_city@6 as ca_city, bought_city@1 as bought_city, ss_ticket_number@0 as ss_ticket_number, amt@2 as amt, profit@3 as profit] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@4, ca_address_sk@0)], filter=bought_city@0 != ca_city@1, projection=[ss_ticket_number@0, bought_city@1, amt@2, profit@3, c_first_name@5, c_last_name@6, ca_city@8] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@1, CAST(customer.c_customer_sk AS Float64)@4)], projection=[ss_ticket_number@0, bought_city@2, amt@3, profit@4, c_current_addr_sk@6, c_first_name@7, c_last_name@8] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, ca_city@3 as bought_city, sum(store_sales.ss_coupon_amt)@4 as amt, sum(store_sales.ss_net_profit)@5 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, ss_addr_sk@2 as ss_addr_sk, ca_city@3 as ca_city], aggr=[sum(store_sales.ss_coupon_amt), sum(store_sales.ss_net_profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_ticket_number@0, ss_customer_sk@1, ss_addr_sk@2, ca_city@3], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_ticket_number@2 as ss_ticket_number, ss_customer_sk@0 as ss_customer_sk, ss_addr_sk@1 as ss_addr_sk, ca_city@5 as ca_city], aggr=[sum(store_sales.ss_coupon_amt), sum(store_sales.ss_net_profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@2)], projection=[ss_customer_sk@0, ss_addr_sk@1, ss_ticket_number@2, ss_coupon_amt@3, ss_net_profit@4, ca_city@6] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_customer_sk@2, ss_addr_sk@4, ss_ticket_number@5, ss_coupon_amt@6, ss_net_profit@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@3)], projection=[ss_customer_sk@2, ss_hdemo_sk@3, ss_addr_sk@4, ss_ticket_number@6, ss_coupon_amt@7, ss_net_profit@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_customer_sk@3, ss_hdemo_sk@4, ss_addr_sk@5, ss_store_sk@6, ss_ticket_number@7, ss_coupon_amt@8, ss_net_profit@9] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_coupon_amt, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_city@1 as ca_city, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_city], file_type=parquet + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_city], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: hd_dep_count@1 = 4 OR hd_vehicle_count@2 = 3, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@1 = 4 OR hd_vehicle_count@2 = 3, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 4 AND 4 <= hd_dep_count_max@1 OR hd_vehicle_count_null_count@6 != row_count@3 AND hd_vehicle_count_min@4 <= 3 AND 3 <= hd_vehicle_count_max@5, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_city@1 = Fairview OR s_city@1 = Midway, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_city], file_type=parquet, predicate=s_city@1 = Fairview OR s_city@1 = Midway, pruning_predicate=s_city_null_count@2 != row_count@3 AND s_city_min@0 <= Fairview AND Fairview <= s_city_max@1 OR s_city_null_count@2 != row_count@3 AND s_city_min@0 <= Midway AND Midway <= s_city_max@1, required_guarantees=[s_city in (Fairview, Midway)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: (d_dow@2 = 6 OR d_dow@2 = 0) AND (d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001), projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_dow], file_type=parquet, predicate=(d_dow@2 = 6 OR d_dow@2 = 0) AND (d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001), pruning_predicate=(d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 6 AND 6 <= d_dow_max@1 OR d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 0 AND 0 <= d_dow_max@1) AND (d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_dow in (0, 6), d_year in (1999, 2000, 2001)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_47() -> Result<()> { + let display = test_tpcds_query(47).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [sum_sales@7 - avg_monthly_sales@6 ASC NULLS LAST, i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_year@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST, avg_monthly_sales@6 ASC NULLS LAST, sum_sales@7 ASC NULLS LAST, psum@8 ASC NULLS LAST, nsum@9 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[sum_sales@7 - avg_monthly_sales@6 ASC NULLS LAST, i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_moy@5 ASC NULLS LAST, avg_monthly_sales@6 ASC NULLS LAST, sum_sales@7 ASC NULLS LAST, psum@8 ASC NULLS LAST, nsum@9 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, d_year@4 as d_year, d_moy@5 as d_moy, avg_monthly_sales@7 as avg_monthly_sales, sum_sales@6 as sum_sales, sum_sales@8 as psum, sum_sales@9 as nsum] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_category@0, i_category@0), (i_brand@1, i_brand@1), (s_store_name@2, s_store_name@2), (s_company_name@3, s_company_name@3), (CAST(v1.rn AS Decimal128(21, 0))@10, v1_lead.rn - Decimal128(Some(1),20,0)@6)], projection=[i_category@0, i_brand@1, s_store_name@2, s_company_name@3, d_year@4, d_moy@5, sum_sales@6, avg_monthly_sales@7, sum_sales@9, sum_sales@15] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, d_year@4 as d_year, d_moy@5 as d_moy, sum_sales@6 as sum_sales, avg_monthly_sales@7 as avg_monthly_sales, rn@8 as rn, sum_sales@9 as sum_sales, CAST(rn@8 AS Decimal128(21, 0)) as CAST(v1.rn AS Decimal128(21, 0))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_category@0, i_category@0), (i_brand@1, i_brand@1), (s_store_name@2, s_store_name@2), (s_company_name@3, s_company_name@3), (CAST(v1.rn AS Decimal128(21, 0))@9, v1_lag.rn + Decimal128(Some(1),20,0)@6)], projection=[i_category@0, i_brand@1, s_store_name@2, s_company_name@3, d_year@4, d_moy@5, sum_sales@6, avg_monthly_sales@7, rn@8, sum_sales@14] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, d_year@4 as d_year, d_moy@5 as d_moy, sum(store_sales.ss_sales_price)@6 as sum_sales, avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 as avg_monthly_sales, rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@8 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@8 AS Decimal128(21, 0)) as CAST(v1.rn AS Decimal128(21, 0))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@4 = 1999 AND avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 > Some(0),19,6 AND CASE WHEN avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 > Some(0),19,6 THEN abs(sum(store_sales.ss_sales_price)@6 - avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7) / avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 END > Some(1000000000),30,10 + │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_year@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3], 3), input_partitions=3 + │ BoundedWindowAggExec: wdw=[avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Decimal128(19, 6) }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_year@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3, d_year@4], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, d_year@4 as d_year, d_moy@5 as d_moy], aggr=[sum(store_sales.ss_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3, d_year@4, d_moy@5], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@1 as i_category, i_brand@0 as i_brand, s_store_name@5 as s_store_name, s_company_name@6 as s_company_name, d_year@3 as d_year, d_moy@4 as d_moy], aggr=[sum(store_sales.ss_sales_price)] + │ ProjectionExec: expr=[i_brand@2 as i_brand, i_category@3 as i_category, ss_sales_price@4 as ss_sales_price, d_year@5 as d_year, d_moy@6 as d_moy, s_store_name@0 as s_store_name, s_company_name@1 as s_company_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@3, ss_store_sk@2)], projection=[s_store_name@1, s_company_name@2, i_brand@4, i_category@5, ss_sales_price@7, d_year@8, d_moy@9] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[i_brand@2 as i_brand, i_category@3 as i_category, ss_store_sk@4 as ss_store_sk, ss_sales_price@5 as ss_sales_price, d_year@0 as d_year, d_moy@1 as d_moy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@2)], projection=[d_year@1, d_moy@2, i_brand@4, i_category@5, ss_store_sk@7, ss_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand@1, i_category@2, ss_sold_date_sk@3, ss_store_sk@5, ss_sales_price@6] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, sum(store_sales.ss_sales_price)@6 as sum_sales, rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 AS Decimal128(20, 0)) + Some(1),20,0 as v1_lag.rn + Decimal128(Some(1),20,0)] + │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_year@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, d_year@4 as d_year, d_moy@5 as d_moy], aggr=[sum(store_sales.ss_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3, d_year@4, d_moy@5], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@1 as i_category, i_brand@0 as i_brand, s_store_name@5 as s_store_name, s_company_name@6 as s_company_name, d_year@3 as d_year, d_moy@4 as d_moy], aggr=[sum(store_sales.ss_sales_price)] + │ ProjectionExec: expr=[i_brand@2 as i_brand, i_category@3 as i_category, ss_sales_price@4 as ss_sales_price, d_year@5 as d_year, d_moy@6 as d_moy, s_store_name@0 as s_store_name, s_company_name@1 as s_company_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@3, ss_store_sk@2)], projection=[s_store_name@1, s_company_name@2, i_brand@4, i_category@5, ss_sales_price@7, d_year@8, d_moy@9] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[i_brand@2 as i_brand, i_category@3 as i_category, ss_store_sk@4 as ss_store_sk, ss_sales_price@5 as ss_sales_price, d_year@0 as d_year, d_moy@1 as d_moy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@2)], projection=[d_year@1, d_moy@2, i_brand@4, i_category@5, ss_store_sk@7, ss_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand@1, i_category@2, ss_sold_date_sk@3, ss_store_sk@5, ss_sales_price@6] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, sum(store_sales.ss_sales_price)@6 as sum_sales, rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 AS Decimal128(20, 0)) - Some(1),20,0 as v1_lead.rn - Decimal128(Some(1),20,0)] + │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_year@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, d_year@4 as d_year, d_moy@5 as d_moy], aggr=[sum(store_sales.ss_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3, d_year@4, d_moy@5], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@1 as i_category, i_brand@0 as i_brand, s_store_name@5 as s_store_name, s_company_name@6 as s_company_name, d_year@3 as d_year, d_moy@4 as d_moy], aggr=[sum(store_sales.ss_sales_price)] + │ ProjectionExec: expr=[i_brand@2 as i_brand, i_category@3 as i_category, ss_sales_price@4 as ss_sales_price, d_year@5 as d_year, d_moy@6 as d_moy, s_store_name@0 as s_store_name, s_company_name@1 as s_company_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@3, ss_store_sk@2)], projection=[s_store_name@1, s_company_name@2, i_brand@4, i_category@5, ss_sales_price@7, d_year@8, d_moy@9] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[i_brand@2 as i_brand, i_category@3 as i_category, ss_store_sk@4 as ss_store_sk, ss_sales_price@5 as ss_sales_price, d_year@0 as d_year, d_moy@1 as d_moy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@2)], projection=[d_year@1, d_moy@2, i_brand@4, i_category@5, ss_store_sk@7, ss_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand@1, i_category@2, ss_sold_date_sk@3, ss_store_sk@5, ss_sales_price@6] + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_name@1 as s_store_name, s_company_name@2 as s_company_name, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_company_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_moy@2 as d_moy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_name@1 as s_store_name, s_company_name@2 as s_company_name, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_company_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_moy@2 as d_moy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_name@1 as s_store_name, s_company_name@2 as s_company_name, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_company_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_moy@2 as d_moy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_48() -> Result<()> { + let display = test_tpcds_query(48).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(store_sales.ss_quantity)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(store_sales.ss_quantity)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_sold_date_sk@0, CAST(date_dim.d_date_sk AS Float64)@1)], projection=[ss_quantity@1] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@2)], filter=(ca_state@1 = CO OR ca_state@1 = OH OR ca_state@1 = TX) AND ss_net_profit@0 >= Some(0),6,2 AND ss_net_profit@0 <= Some(200000),6,2 OR (ca_state@1 = OR OR ca_state@1 = MN OR ca_state@1 = KY) AND ss_net_profit@0 >= Some(15000),6,2 AND ss_net_profit@0 <= Some(300000),6,2 OR (ca_state@1 = VA OR ca_state@1 = CA OR ca_state@1 = MS) AND ss_net_profit@0 >= Some(5000),6,2 AND CAST(ss_net_profit@0 AS Decimal128(22, 2)) <= Some(2500000),22,2, projection=[ss_sold_date_sk@0, ss_quantity@2] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_cdemo_sk@1, CAST(customer_demographics.cd_demo_sk AS Float64)@3)], filter=cd_marital_status@1 = M AND cd_education_status@2 = 4 yr Degree AND ss_sales_price@0 >= Some(10000),5,2 AND ss_sales_price@0 <= Some(15000),5,2 OR cd_marital_status@1 = D AND cd_education_status@2 = 2 yr Degree AND ss_sales_price@0 >= Some(5000),5,2 AND ss_sales_price@0 <= Some(10000),5,2 OR cd_marital_status@1 = S AND cd_education_status@2 = College AND ss_sales_price@0 >= Some(15000),5,2 AND ss_sales_price@0 <= Some(20000),5,2, projection=[ss_sold_date_sk@0, ss_addr_sk@2, ss_quantity@3, ss_net_profit@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@3)], projection=[ss_sold_date_sk@2, ss_cdemo_sk@3, ss_addr_sk@4, ss_quantity@6, ss_sales_price@7, ss_net_profit@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: (ss_net_profit@6 >= Some(0),6,2 AND ss_net_profit@6 <= Some(200000),6,2 OR ss_net_profit@6 >= Some(15000),6,2 AND ss_net_profit@6 <= Some(300000),6,2 OR ss_net_profit@6 >= Some(5000),6,2 AND CAST(ss_net_profit@6 AS Decimal128(22, 2)) <= Some(2500000),22,2) AND (ss_sales_price@5 >= Some(10000),5,2 AND ss_sales_price@5 <= Some(15000),5,2 OR ss_sales_price@5 >= Some(5000),5,2 AND ss_sales_price@5 <= Some(10000),5,2 OR ss_sales_price@5 >= Some(15000),5,2 AND ss_sales_price@5 <= Some(20000),5,2) + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_cdemo_sk, ss_addr_sk, ss_store_sk, ss_quantity, ss_sales_price, ss_net_profit], file_type=parquet, predicate=(ss_net_profit@6 >= Some(0),6,2 AND ss_net_profit@6 <= Some(200000),6,2 OR ss_net_profit@6 >= Some(15000),6,2 AND ss_net_profit@6 <= Some(300000),6,2 OR ss_net_profit@6 >= Some(5000),6,2 AND CAST(ss_net_profit@6 AS Decimal128(22, 2)) <= Some(2500000),22,2) AND (ss_sales_price@5 >= Some(10000),5,2 AND ss_sales_price@5 <= Some(15000),5,2 OR ss_sales_price@5 >= Some(5000),5,2 AND ss_sales_price@5 <= Some(10000),5,2 OR ss_sales_price@5 >= Some(15000),5,2 AND ss_sales_price@5 <= Some(20000),5,2) AND DynamicFilter [ empty ], pruning_predicate=(ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(0),6,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(200000),6,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(15000),6,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(300000),6,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(5000),6,2 AND ss_net_profit_null_count@1 != row_count@2 AND CAST(ss_net_profit_min@3 AS Decimal128(22, 2)) <= Some(2500000),22,2) AND (ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(10000),5,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(15000),5,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(5000),5,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(10000),5,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(15000),5,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(20000),5,2), required_guarantees=[] + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cd_marital_status@1 = M AND cd_education_status@2 = 4 yr Degree OR cd_marital_status@1 = D AND cd_education_status@2 = 2 yr Degree OR cd_marital_status@1 = S AND cd_education_status@2 = College + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_marital_status@1 = M AND cd_education_status@2 = 4 yr Degree OR cd_marital_status@1 = D AND cd_education_status@2 = 2 yr Degree OR cd_marital_status@1 = S AND cd_education_status@2 = College, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 4 yr Degree AND 4 yr Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= D AND D <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, 4 yr Degree, College), cd_marital_status in (D, M, S)] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_state@1 as ca_state, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: (ca_state@1 = CO OR ca_state@1 = OH OR ca_state@1 = TX OR ca_state@1 = OR OR ca_state@1 = MN OR ca_state@1 = KY OR ca_state@1 = VA OR ca_state@1 = CA OR ca_state@1 = MS) AND ca_state@1 IN ([CO, OH, TX, OR, MN, KY, VA, CA, MS]) AND ca_country@2 = United States, projection=[ca_address_sk@0, ca_state@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=(ca_state@1 = CO OR ca_state@1 = OH OR ca_state@1 = TX OR ca_state@1 = OR OR ca_state@1 = MN OR ca_state@1 = KY OR ca_state@1 = VA OR ca_state@1 = CA OR ca_state@1 = MS) AND ca_state@1 IN ([CO, OH, TX, OR, MN, KY, VA, CA, MS]) AND ca_country@2 = United States, pruning_predicate=(ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= CO AND CO <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OH AND OH <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= TX AND TX <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OR AND OR <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MN AND MN <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= KY AND KY <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= VA AND VA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= CA AND CA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1) AND (ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= CO AND CO <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OH AND OH <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= TX AND TX <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OR AND OR <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MN AND MN <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= KY AND KY <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= VA AND VA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= CA AND CA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1) AND ca_country_null_count@6 != row_count@3 AND ca_country_min@4 <= United States AND United States <= ca_country_max@5, required_guarantees=[ca_country in (United States), ca_state in (CA, CO, KY, MN, MS, OH, OR, TX, VA)] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_49() -> Result<()> { + let display = test_tpcds_query(49).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [channel@0 ASC, return_rank@3 ASC, currency_rank@4 ASC, item@1 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, return_rank@3 ASC, currency_rank@4 ASC, item@1 ASC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, item@1 as item, return_ratio@2 as return_ratio, return_rank@3 as return_rank, currency_rank@4 as currency_rank], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([channel@0, item@1, return_ratio@2, return_rank@3, currency_rank@4], 3), input_partitions=9 + │ AggregateExec: mode=Partial, gby=[channel@0 as channel, item@1 as item, return_ratio@2 as return_ratio, return_rank@3 as return_rank, currency_rank@4 as currency_rank], aggr=[], ordering_mode=PartiallySorted([0, 4]) + │ UnionExec + │ ProjectionExec: expr=[web as channel, item@0 as item, return_ratio@1 as return_ratio, return_rank@2 as return_rank, currency_rank@3 as currency_rank] + │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as return_rank, rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as currency_rank] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 <= 10 OR rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 <= 10 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1 + │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 as rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[currency_ratio@2 ASC NULLS LAST], preserve_partitioning=[false] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortPreservingMergeExec: [return_ratio@1 ASC NULLS LAST] + │ SortExec: expr=[return_ratio@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[ws_item_sk@0 as item, CAST(sum(coalesce(wr.wr_return_quantity,Int64(0)))@1 AS Decimal128(15, 4)) / CAST(sum(coalesce(ws.ws_quantity,Int64(0)))@2 AS Decimal128(15, 4)) as return_ratio, CAST(sum(coalesce(wr.wr_return_amt,Int64(0)))@3 AS Decimal128(15, 4)) / CAST(sum(coalesce(ws.ws_net_paid,Int64(0)))@4 AS Decimal128(15, 4)) as currency_ratio] + │ AggregateExec: mode=FinalPartitioned, gby=[ws_item_sk@0 as ws_item_sk], aggr=[sum(coalesce(wr.wr_return_quantity,Int64(0))), sum(coalesce(ws.ws_quantity,Int64(0))), sum(coalesce(wr.wr_return_amt,Int64(0))), sum(coalesce(ws.ws_net_paid,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_item_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ws_item_sk@0 as ws_item_sk], aggr=[sum(coalesce(wr.wr_return_quantity,Int64(0))), sum(coalesce(ws.ws_quantity,Int64(0))), sum(coalesce(wr.wr_return_amt,Int64(0))), sum(coalesce(ws.ws_net_paid,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_quantity@4, ws_net_paid@5, wr_return_quantity@6, wr_return_amt@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: wr_return_amt@5 > Some(1000000),7,2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(ws_order_number@2, wr_order_number@1), (ws_item_sk@1, wr_item_sk@0)], projection=[ws_sold_date_sk@0, ws_item_sk@1, ws_quantity@3, ws_net_paid@4, wr_return_quantity@7, wr_return_amt@8] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[catalog as channel, item@0 as item, return_ratio@1 as return_ratio, return_rank@2 as return_rank, currency_rank@3 as currency_rank] + │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as return_rank, rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as currency_rank] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 <= 10 OR rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 <= 10 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1 + │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 as rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[currency_ratio@2 ASC NULLS LAST], preserve_partitioning=[false] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortPreservingMergeExec: [return_ratio@1 ASC NULLS LAST] + │ SortExec: expr=[return_ratio@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[cs_item_sk@0 as item, CAST(sum(coalesce(cr.cr_return_quantity,Int64(0)))@1 AS Decimal128(15, 4)) / CAST(sum(coalesce(cs.cs_quantity,Int64(0)))@2 AS Decimal128(15, 4)) as return_ratio, CAST(sum(coalesce(cr.cr_return_amount,Int64(0)))@3 AS Decimal128(15, 4)) / CAST(sum(coalesce(cs.cs_net_paid,Int64(0)))@4 AS Decimal128(15, 4)) as currency_ratio] + │ AggregateExec: mode=FinalPartitioned, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(coalesce(cr.cr_return_quantity,Int64(0))), sum(coalesce(cs.cs_quantity,Int64(0))), sum(coalesce(cr.cr_return_amount,Int64(0))), sum(coalesce(cs.cs_net_paid,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(coalesce(cr.cr_return_quantity,Int64(0))), sum(coalesce(cs.cs_quantity,Int64(0))), sum(coalesce(cr.cr_return_amount,Int64(0))), sum(coalesce(cs.cs_net_paid,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_item_sk@3, cs_quantity@4, cs_net_paid@5, cr_return_quantity@6, cr_return_amount@7] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cr_return_amount@5 > Some(1000000),7,2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(cs_order_number@2, cr_order_number@1), (cs_item_sk@1, cr_item_sk@0)], projection=[cs_sold_date_sk@0, cs_item_sk@1, cs_quantity@3, cs_net_paid@4, cr_return_quantity@7, cr_return_amount@8] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[store as channel, item@0 as item, return_ratio@1 as return_ratio, return_rank@2 as return_rank, currency_rank@3 as currency_rank] + │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as return_rank, rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as currency_rank] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 <= 10 OR rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 <= 10 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1 + │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 as rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[currency_ratio@2 ASC NULLS LAST], preserve_partitioning=[false] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortPreservingMergeExec: [return_ratio@1 ASC NULLS LAST] + │ SortExec: expr=[return_ratio@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[ss_item_sk@0 as item, CAST(sum(coalesce(sr.sr_return_quantity,Int64(0)))@1 AS Decimal128(15, 4)) / CAST(sum(coalesce(sts.ss_quantity,Int64(0)))@2 AS Decimal128(15, 4)) as return_ratio, CAST(sum(coalesce(sr.sr_return_amt,Int64(0)))@3 AS Decimal128(15, 4)) / CAST(sum(coalesce(sts.ss_net_paid,Int64(0)))@4 AS Decimal128(15, 4)) as currency_ratio] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_item_sk@0 as ss_item_sk], aggr=[sum(coalesce(sr.sr_return_quantity,Int64(0))), sum(coalesce(sts.ss_quantity,Int64(0))), sum(coalesce(sr.sr_return_amt,Int64(0))), sum(coalesce(sts.ss_net_paid,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_item_sk@0 as ss_item_sk], aggr=[sum(coalesce(sr.sr_return_quantity,Int64(0))), sum(coalesce(sts.ss_quantity,Int64(0))), sum(coalesce(sr.sr_return_amt,Int64(0))), sum(coalesce(sts.ss_net_paid,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_quantity@4, ss_net_paid@5, sr_return_quantity@6, sr_return_amt@7] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: sr_return_amt@5 > Some(1000000),7,2 + │ ProjectionExec: expr=[ss_sold_date_sk@2 as ss_sold_date_sk, ss_item_sk@3 as ss_item_sk, ss_quantity@4 as ss_quantity, ss_net_paid@5 as ss_net_paid, sr_return_quantity@0 as sr_return_quantity, sr_return_amt@1 as sr_return_amt] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(sr_ticket_number@1, ss_ticket_number@2), (sr_item_sk@0, ss_item_sk@1)], projection=[sr_return_quantity@2, sr_return_amt@3, ss_sold_date_sk@4, ss_item_sk@5, ss_quantity@7, ss_net_paid@8] + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 12, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_order_number@2, ws_item_sk@1], 3), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ws_net_profit@5 > Some(100),7,2 AND ws_net_paid@4 > Some(0),7,2 AND ws_quantity@3 > 0, projection=[ws_sold_date_sk@0, ws_item_sk@1, ws_order_number@2, ws_quantity@3, ws_net_paid@4] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_net_paid, ws_net_profit], file_type=parquet, predicate=ws_net_profit@5 > Some(100),7,2 AND ws_net_paid@4 > Some(0),7,2 AND ws_quantity@3 > 0, pruning_predicate=ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 > Some(100),7,2 AND ws_net_paid_null_count@4 != row_count@2 AND ws_net_paid_max@3 > Some(0),7,2 AND ws_quantity_null_count@6 != row_count@2 AND ws_quantity_max@5 > 0, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([wr_order_number@1, wr_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 12, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_order_number@2, cs_item_sk@1], 3), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cs_net_profit@5 > Some(100),7,2 AND cs_net_paid@4 > Some(0),7,2 AND cs_quantity@3 > 0, projection=[cs_sold_date_sk@0, cs_item_sk@1, cs_order_number@2, cs_quantity@3, cs_net_paid@4] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_net_paid, cs_net_profit], file_type=parquet, predicate=cs_net_profit@5 > Some(100),7,2 AND cs_net_paid@4 > Some(0),7,2 AND cs_quantity@3 > 0, pruning_predicate=cs_net_profit_null_count@1 != row_count@2 AND cs_net_profit_max@0 > Some(100),7,2 AND cs_net_paid_null_count@4 != row_count@2 AND cs_net_paid_max@3 > Some(0),7,2 AND cs_quantity_null_count@6 != row_count@2 AND cs_quantity_max@5 > 0, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cr_order_number@1, cr_item_sk@0], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 12, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_ticket_number@2, ss_item_sk@1], 3), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_net_profit@5 > Some(100),6,2 AND ss_net_paid@4 > Some(0),7,2 AND ss_quantity@3 > 0, projection=[ss_sold_date_sk@0, ss_item_sk@1, ss_ticket_number@2, ss_quantity@3, ss_net_paid@4] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_net_paid, ss_net_profit], file_type=parquet, predicate=ss_net_profit@5 > Some(100),6,2 AND ss_net_paid@4 > Some(0),7,2 AND ss_quantity@3 > 0, pruning_predicate=ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 > Some(100),6,2 AND ss_net_paid_null_count@4 != row_count@2 AND ss_net_paid_max@3 > Some(0),7,2 AND ss_quantity_null_count@6 != row_count@2 AND ss_quantity_max@5 > 0, required_guarantees=[] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_50() -> Result<()> { + let display = test_tpcds_query(50).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [s_store_name@0 ASC NULLS LAST, s_company_id@1 ASC NULLS LAST, s_street_number@2 ASC NULLS LAST, s_street_name@3 ASC NULLS LAST, s_street_type@4 ASC NULLS LAST, s_suite_number@5 ASC NULLS LAST, s_city@6 ASC NULLS LAST, s_county@7 ASC NULLS LAST, s_state@8 ASC NULLS LAST, s_zip@9 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[s_store_name@0 ASC NULLS LAST, s_company_id@1 ASC NULLS LAST, s_street_number@2 ASC NULLS LAST, s_street_name@3 ASC NULLS LAST, s_street_type@4 ASC NULLS LAST, s_suite_number@5 ASC NULLS LAST, s_city@6 ASC NULLS LAST, s_county@7 ASC NULLS LAST, s_state@8 ASC NULLS LAST, s_zip@9 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[s_store_name@0 as s_store_name, s_company_id@1 as s_company_id, s_street_number@2 as s_street_number, s_street_name@3 as s_street_name, s_street_type@4 as s_street_type, s_suite_number@5 as s_suite_number, s_city@6 as s_city, s_county@7 as s_county, s_state@8 as s_state, s_zip@9 as s_zip, sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END)@10 as 30 days, sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(30) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END)@11 as 31-60 days, sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(60) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END)@12 as 61-90 days, sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(90) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END)@13 as 91-120 days, sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)@14 as >120 days] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_name@0 as s_store_name, s_company_id@1 as s_company_id, s_street_number@2 as s_street_number, s_street_name@3 as s_street_name, s_street_type@4 as s_street_type, s_suite_number@5 as s_suite_number, s_city@6 as s_city, s_county@7 as s_county, s_state@8 as s_state, s_zip@9 as s_zip], aggr=[sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(30) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(60) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(90) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([s_store_name@0, s_company_id@1, s_street_number@2, s_street_name@3, s_street_type@4, s_suite_number@5, s_city@6, s_county@7, s_state@8, s_zip@9], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[s_store_name@1 as s_store_name, s_company_id@2 as s_company_id, s_street_number@3 as s_street_number, s_street_name@4 as s_street_name, s_street_type@5 as s_street_type, s_suite_number@6 as s_suite_number, s_city@7 as s_city, s_county@8 as s_county, s_state@9 as s_state, s_zip@10 as s_zip], aggr=[sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(30) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(60) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(90) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)] + │ ProjectionExec: expr=[sr_returned_date_sk@1 - ss_sold_date_sk@0 as __common_expr_1, s_store_name@2 as s_store_name, s_company_id@3 as s_company_id, s_street_number@4 as s_street_number, s_street_name@5 as s_street_name, s_street_type@6 as s_street_type, s_suite_number@7 as s_suite_number, s_city@8 as s_city, s_county@9 as s_county, s_state@10 as s_state, s_zip@11 as s_zip] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(sr_returned_date_sk@1, CAST(d2.d_date_sk AS Float64)@1)], projection=[ss_sold_date_sk@0, sr_returned_date_sk@1, s_store_name@2, s_company_id@3, s_street_number@4, s_street_name@5, s_street_type@6, s_suite_number@7, s_city@8, s_county@9, s_state@10, s_zip@11] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_sold_date_sk@0, CAST(d1.d_date_sk AS Float64)@1)], projection=[ss_sold_date_sk@0, sr_returned_date_sk@1, s_store_name@2, s_company_id@3, s_street_number@4, s_street_name@5, s_street_type@6, s_suite_number@7, s_city@8, s_county@9, s_state@10, s_zip@11] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_sold_date_sk@10 as ss_sold_date_sk, sr_returned_date_sk@11 as sr_returned_date_sk, s_store_name@0 as s_store_name, s_company_id@1 as s_company_id, s_street_number@2 as s_street_number, s_street_name@3 as s_street_name, s_street_type@4 as s_street_type, s_suite_number@5 as s_suite_number, s_city@6 as s_city, s_county@7 as s_county, s_state@8 as s_state, s_zip@9 as s_zip] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@11, ss_store_sk@1)], projection=[s_store_name@1, s_company_id@2, s_street_number@3, s_street_name@4, s_street_type@5, s_suite_number@6, s_city@7, s_county@8, s_state@9, s_zip@10, ss_sold_date_sk@12, sr_returned_date_sk@14] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@1 as ss_sold_date_sk, ss_store_sk@2 as ss_store_sk, sr_returned_date_sk@0 as sr_returned_date_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_ticket_number@3, ss_ticket_number@4), (sr_item_sk@1, ss_item_sk@1), (sr_customer_sk@2, ss_customer_sk@2)], projection=[sr_returned_date_sk@0, ss_sold_date_sk@4, ss_store_sk@7] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk], file_type=parquet + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 8, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 8, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 8 AND 8 <= d_moy_max@5, required_guarantees=[d_moy in (8), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_name@1 as s_store_name, s_company_id@2 as s_company_id, s_street_number@3 as s_street_number, s_street_name@4 as s_street_name, s_street_type@5 as s_street_type, s_suite_number@6 as s_suite_number, s_city@7 as s_city, s_county@8 as s_county, s_state@9 as s_state, s_zip@10 as s_zip, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_ticket_number@3, sr_item_sk@1, sr_customer_sk@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_ticket_number@4, ss_item_sk@1, ss_customer_sk@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_51() -> Result<()> { + let display = test_tpcds_query(51).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [item_sk@0 ASC, d_date@1 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[item_sk@0 ASC, d_date@1 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[item_sk@0 as item_sk, d_date@1 as d_date, web_sales@2 as web_sales, store_sales@3 as store_sales, max(x.web_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 as web_cumulative, max(x.store_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@5 as store_cumulative] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: max(x.web_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 > max(x.store_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@5 + │ BoundedWindowAggExec: wdw=[max(x.web_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "max(x.web_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Decimal128(25, 2) }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, max(x.store_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "max(x.store_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Decimal128(25, 2) }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[item_sk@0 ASC NULLS LAST, d_date@1 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([item_sk@0], 3), input_partitions=3 + │ ProjectionExec: expr=[CASE WHEN item_sk@0 IS NOT NULL THEN item_sk@0 ELSE item_sk@3 END as item_sk, CASE WHEN d_date@1 IS NOT NULL THEN d_date@1 ELSE d_date@4 END as d_date, cume_sales@2 as web_sales, cume_sales@5 as store_sales] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Full, on=[(item_sk@0, item_sk@0), (d_date@1, d_date@1)] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ws_item_sk@0 as item_sk, d_date@1 as d_date, sum(sum(web_sales.ws_sales_price)) PARTITION BY [web_sales.ws_item_sk] ORDER BY [date_dim.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as cume_sales] + │ BoundedWindowAggExec: wdw=[sum(sum(web_sales.ws_sales_price)) PARTITION BY [web_sales.ws_item_sk] ORDER BY [date_dim.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(sum(web_sales.ws_sales_price)) PARTITION BY [web_sales.ws_item_sk] ORDER BY [date_dim.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Decimal128(25, 2) }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[ws_item_sk@0 ASC NULLS LAST, d_date@1 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_item_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[ws_item_sk@0 as ws_item_sk, d_date@1 as d_date], aggr=[sum(web_sales.ws_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_item_sk@0, d_date@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ws_item_sk@0 as ws_item_sk, d_date@2 as d_date], aggr=[sum(web_sales.ws_sales_price)] + │ ProjectionExec: expr=[ws_item_sk@1 as ws_item_sk, ws_sales_price@2 as ws_sales_price, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@0)], projection=[d_date@1, ws_item_sk@4, ws_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ws_item_sk@1 IS NOT NULL + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_sales_price], file_type=parquet, predicate=ws_item_sk@1 IS NOT NULL AND DynamicFilter [ empty ], pruning_predicate=ws_item_sk_null_count@1 != row_count@0, required_guarantees=[] + │ ProjectionExec: expr=[ss_item_sk@0 as item_sk, d_date@1 as d_date, sum(sum(store_sales.ss_sales_price)) PARTITION BY [store_sales.ss_item_sk] ORDER BY [date_dim.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as cume_sales] + │ BoundedWindowAggExec: wdw=[sum(sum(store_sales.ss_sales_price)) PARTITION BY [store_sales.ss_item_sk] ORDER BY [date_dim.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(sum(store_sales.ss_sales_price)) PARTITION BY [store_sales.ss_item_sk] ORDER BY [date_dim.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Decimal128(25, 2) }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[ss_item_sk@0 ASC NULLS LAST, d_date@1 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[ss_item_sk@0 as ss_item_sk, d_date@1 as d_date], aggr=[sum(store_sales.ss_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@0, d_date@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_item_sk@0 as ss_item_sk, d_date@2 as d_date], aggr=[sum(store_sales.ss_sales_price)] + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_sales_price@2 as ss_sales_price, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_date@1, ss_item_sk@4, ss_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_item_sk@1 IS NOT NULL + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_sales_price], file_type=parquet, predicate=ss_item_sk@1 IS NOT NULL AND DynamicFilter [ empty ], pruning_predicate=ss_item_sk_null_count@1 != row_count@0, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_52() -> Result<()> { + let display = test_tpcds_query(52).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [d_year@0 ASC NULLS LAST, ext_price@3 DESC, brand_id@1 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[ext_price@3 DESC, brand_id@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[d_year@0 as d_year, i_brand_id@2 as brand_id, i_brand@1 as brand, sum(store_sales.ss_ext_sales_price)@3 as ext_price] + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand@1 as i_brand, i_brand_id@2 as i_brand_id], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([0]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([d_year@0, i_brand@1, i_brand_id@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand@3 as i_brand, i_brand_id@2 as i_brand_id], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([0]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@1, i_item_sk@0)], projection=[d_year@0, ss_ext_sales_price@2, i_brand_id@4, i_brand@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(dt.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_manager_id@3 = 1, projection=[i_item_sk@0, i_brand_id@1, i_brand@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@3 = 1 AND DynamicFilter [ empty ], pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(dt.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_moy@2 = 11 AND d_year@1 = 2000, projection=[d_date_sk@0, d_year@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 11 AND d_year@1 = 2000, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (2000)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_53() -> Result<()> { + let display = test_tpcds_query(53).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [avg_quarterly_sales@2 ASC NULLS LAST, sum_sales@1 ASC NULLS LAST, i_manufact_id@0 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[avg_quarterly_sales@2 ASC NULLS LAST, sum_sales@1 ASC NULLS LAST, i_manufact_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(store_sales.ss_sales_price)@1 as sum_sales, avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 as avg_quarterly_sales] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CASE WHEN avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 > Some(0),19,6 THEN abs(sum(store_sales.ss_sales_price)@1 - avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2) / avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 END > Some(1000000000),30,10 + │ WindowAggExec: wdw=[avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Decimal128(19, 6), nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + │ SortExec: expr=[i_manufact_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_manufact_id@0], 3), input_partitions=3 + │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(store_sales.ss_sales_price)@2 as sum(store_sales.ss_sales_price)] + │ AggregateExec: mode=FinalPartitioned, gby=[i_manufact_id@0 as i_manufact_id, d_qoy@1 as d_qoy], aggr=[sum(store_sales.ss_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_manufact_id@0, d_qoy@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_manufact_id@0 as i_manufact_id, d_qoy@2 as d_qoy], aggr=[sum(store_sales.ss_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@1)], projection=[i_manufact_id@2, ss_sales_price@4, d_qoy@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[i_manufact_id@1 as i_manufact_id, ss_store_sk@2 as ss_store_sk, ss_sales_price@3 as ss_sales_price, d_qoy@0 as d_qoy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@1)], projection=[d_qoy@1, i_manufact_id@3, ss_store_sk@5, ss_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_manufact_id@1, ss_sold_date_sk@2, ss_store_sk@4, ss_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_qoy@1 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_month_seq@1 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), projection=[d_date_sk@0, d_qoy@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq, d_qoy], file_type=parquet, predicate=d_month_seq@1 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), pruning_predicate=d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1200 AND 1200 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1201 AND 1201 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1202 AND 1202 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1203 AND 1203 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1204 AND 1204 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1205 AND 1205 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1206 AND 1206 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1207 AND 1207 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1208 AND 1208 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1209 AND 1209 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1210 AND 1210 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1211 AND 1211 <= d_month_seq_max@1, required_guarantees=[d_month_seq in (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: (i_category@3 = Books OR i_category@3 = Children OR i_category@3 = Electronics) AND i_class@2 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@1 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@3 = Women OR i_category@3 = Music OR i_category@3 = Men) AND i_class@2 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@1 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), projection=[i_item_sk@0, i_manufact_id@4] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_manufact_id], file_type=parquet, predicate=(i_category@3 = Books OR i_category@3 = Children OR i_category@3 = Electronics) AND i_class@2 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@1 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@3 = Women OR i_category@3 = Music OR i_category@3 = Men) AND i_class@2 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@1 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Children AND Children <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= personal AND personal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= portable AND portable <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= reference AND reference <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= self-help AND self-help <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #14 AND scholaramalgamalg #14 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #7 AND scholaramalgamalg #7 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiunivamalg #9 AND exportiunivamalg #9 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #9 AND scholaramalgamalg #9 <= i_brand_max@8) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= accessories AND accessories <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= classical AND classical <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= fragrances AND fragrances <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= pants AND pants <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= amalgimporto #1 AND amalgimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= edu packscholar #1 AND edu packscholar #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiimporto #1 AND exportiimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= importoamalg #1 AND importoamalg #1 <= i_brand_max@8), required_guarantees=[i_brand in (amalgimporto #1, edu packscholar #1, exportiimporto #1, exportiunivamalg #9, importoamalg #1, scholaramalgamalg #14, scholaramalgamalg #7, scholaramalgamalg #9), i_class in (accessories, classical, fragrances, pants, personal, portable, reference, self-help)] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_54() -> Result<()> { + let display = test_tpcds_query(54).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [segment@0 ASC, num_customers@1 ASC, segment_base@2 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[segment@0 ASC, num_customers@1 ASC, segment_base@2 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[segment@0 as segment, count(Int64(1))@1 as num_customers, CAST(segment@0 AS Int64) * 50 as segment_base] + │ AggregateExec: mode=FinalPartitioned, gby=[segment@0 as segment], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([segment@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[segment@0 as segment], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[CAST(round(CAST(sum(store_sales.ss_ext_sales_price)@1 / Some(50),20,0 AS Float64)) AS Int32) as segment] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_ext_sales_price)] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, ss_ext_sales_price@1 as ss_ext_sales_price] + │ NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_2@1 <= date_dim.d_month_seq + Int64(3)@0, projection=[c_customer_sk@0, ss_ext_sales_price@1, d_month_seq@2, date_dim.d_month_seq + Int64(3)@4] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, ss_ext_sales_price@1 as ss_ext_sales_price, d_month_seq@2 as d_month_seq, CAST(d_month_seq@2 AS Int64) as join_proj_push_down_2] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[c_customer_sk@1 as c_customer_sk, ss_ext_sales_price@2 as ss_ext_sales_price, d_month_seq@3 as d_month_seq] + │ NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 >= date_dim.d_month_seq + Int64(1)@0, projection=[date_dim.d_month_seq + Int64(1)@0, c_customer_sk@1, ss_ext_sales_price@2, d_month_seq@3] + │ CoalescePartitionsExec + │ AggregateExec: mode=FinalPartitioned, gby=[date_dim.d_month_seq + Int64(1)@0 as date_dim.d_month_seq + Int64(1)], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, ss_ext_sales_price@1 as ss_ext_sales_price, d_month_seq@2 as d_month_seq, CAST(d_month_seq@2 AS Int64) as join_proj_push_down_1] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_sold_date_sk@1, CAST(date_dim.d_date_sk AS Float64)@2)], projection=[c_customer_sk@0, ss_ext_sales_price@2, d_month_seq@4] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_county@0, ca_county@3), (CAST(store.s_state AS Utf8View)@2, ca_state@4)], projection=[c_customer_sk@3, ss_sold_date_sk@4, ss_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@1, ca_address_sk@0)], projection=[c_customer_sk@0, ss_sold_date_sk@2, ss_ext_sales_price@3, ca_county@5, ca_state@6] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(my_customers.c_customer_sk AS Float64)@2, ss_customer_sk@1)], projection=[c_customer_sk@0, c_current_addr_sk@1, ss_sold_date_sk@3, ss_ext_sales_price@5] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, CAST(c_customer_sk@0 AS Float64) as CAST(my_customers.c_customer_sk AS Float64)] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_customer_sk@0, c_current_addr_sk@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_sk@0, CAST(customer.c_customer_sk AS Float64)@2)], projection=[c_customer_sk@1, c_current_addr_sk@2] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, sold_date_sk@0)], projection=[customer_sk@3] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, item_sk@2)], projection=[sold_date_sk@1, customer_sk@2] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ UnionExec + │ ProjectionExec: expr=[cs_sold_date_sk@0 as sold_date_sk, cs_bill_customer_sk@1 as customer_sk, cs_item_sk@2 as item_sk] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk], file_type=parquet + │ ProjectionExec: expr=[ws_sold_date_sk@0 as sold_date_sk, ws_bill_customer_sk@2 as customer_sk, ws_item_sk@1 as item_sk] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk], file_type=parquet + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_county, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_month_seq@1 as d_month_seq, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq], file_type=parquet + │ AggregateExec: mode=FinalPartitioned, gby=[date_dim.d_month_seq + Int64(3)@0 as date_dim.d_month_seq + Int64(3)], aggr=[] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([date_dim.d_month_seq + Int64(1)@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[date_dim.d_month_seq + Int64(1)@0 as date_dim.d_month_seq + Int64(1)], aggr=[] + │ ProjectionExec: expr=[CAST(d_month_seq@0 AS Int64) + 1 as date_dim.d_month_seq + Int64(1)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 12, projection=[d_month_seq@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 1998 AND d_moy@2 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_county@0 as s_county, s_state@1 as s_state, CAST(s_state@1 AS Utf8View) as CAST(store.s_state AS Utf8View)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_county, s_state], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_moy@2 = 12 AND d_year@1 = 1998, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 12 AND d_year@1 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 12 AND 12 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_category@2 = Women AND i_class@1 = maternity, projection=[i_item_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=i_category@2 = Women AND i_class@1 = maternity, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND i_class_null_count@6 != row_count@3 AND i_class_min@4 <= maternity AND maternity <= i_class_max@5, required_guarantees=[i_category in (Women), i_class in (maternity)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([date_dim.d_month_seq + Int64(3)@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[date_dim.d_month_seq + Int64(3)@0 as date_dim.d_month_seq + Int64(3)], aggr=[] + │ ProjectionExec: expr=[CAST(d_month_seq@0 AS Int64) + 3 as date_dim.d_month_seq + Int64(3)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 12, projection=[d_month_seq@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 1998 AND d_moy@2 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_55() -> Result<()> { + let display = test_tpcds_query(55).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[brand_id@0 as brand_id, brand@1 as brand, ext_price@2 as ext_price] + │ SortPreservingMergeExec: [ext_price@2 DESC, i_brand_id@3 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[ext_price@2 DESC, brand_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_brand_id@1 as brand_id, i_brand@0 as brand, sum(store_sales.ss_ext_sales_price)@2 as ext_price, i_brand_id@1 as i_brand_id] + │ AggregateExec: mode=FinalPartitioned, gby=[i_brand@0 as i_brand, i_brand_id@1 as i_brand_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_brand@0, i_brand_id@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_brand@2 as i_brand, i_brand_id@1 as i_brand_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_ext_sales_price@1, i_brand_id@3, i_brand@4] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_ext_sales_price@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_manager_id@3 = 28, projection=[i_item_sk@0, i_brand_id@1, i_brand@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@3 = 28 AND DynamicFilter [ empty ], pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 28 AND 28 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (28)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1999, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 11 AND d_year@1 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_56() -> Result<()> { + let display = test_tpcds_query(56).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [total_sales@1 ASC, i_item_id@0 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[total_sales@1 ASC, i_item_id@0 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(tmp1.total_sales)@1 as total_sales] + │ AggregateExec: mode=SinglePartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(tmp1.total_sales)] + │ InterleaveExec + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(store_sales.ss_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(i_item_id@1, i_item_id@0)] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_ext_sales_price@1, i_item_id@3] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@1)], projection=[ss_item_sk@0, ss_ext_sales_price@2] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_addr_sk@4, ss_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@1 = Some(-500),4,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_color@1 = slate OR i_color@1 = blanched OR i_color@1 = burnished, projection=[i_item_id@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@1 = slate OR i_color@1 = blanched OR i_color@1 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(catalog_sales.cs_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(i_item_id@1, i_item_id@0)] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, i_item_sk@0)], projection=[cs_ext_sales_price@1, i_item_id@3] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_bill_addr_sk@0, CAST(customer_address.ca_address_sk AS Float64)@1)], projection=[cs_item_sk@1, cs_ext_sales_price@2] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_bill_addr_sk@3, cs_item_sk@4, cs_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@1 = Some(-500),4,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_color@1 = slate OR i_color@1 = blanched OR i_color@1 = burnished, projection=[i_item_id@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@1 = slate OR i_color@1 = blanched OR i_color@1 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(web_sales.ws_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(web_sales.ws_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(web_sales.ws_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(i_item_id@1, i_item_id@0)] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@0, i_item_sk@0)], projection=[ws_ext_sales_price@1, i_item_id@3] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_bill_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@1)], projection=[ws_item_sk@0, ws_ext_sales_price@2] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_bill_addr_sk@4, ws_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@1 = Some(-500),4,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_color@1 = slate OR i_color@1 = blanched OR i_color@1 = burnished, projection=[i_item_id@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@1 = slate OR i_color@1 = blanched OR i_color@1 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 2, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 2, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 2, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_57() -> Result<()> { + let display = test_tpcds_query(57).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [sum_sales@6 - avg_monthly_sales@5 ASC, i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_year@3 ASC NULLS LAST, d_moy@4 ASC NULLS LAST, avg_monthly_sales@5 ASC NULLS LAST, sum_sales@6 ASC NULLS LAST, psum@7 ASC NULLS LAST, nsum@8 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[sum_sales@6 - avg_monthly_sales@5 ASC, i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_moy@4 ASC NULLS LAST, avg_monthly_sales@5 ASC NULLS LAST, sum_sales@6 ASC NULLS LAST, psum@7 ASC NULLS LAST, nsum@8 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy, avg_monthly_sales@6 as avg_monthly_sales, sum_sales@5 as sum_sales, sum_sales@7 as psum, sum_sales@8 as nsum] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_category@0, i_category@0), (i_brand@1, i_brand@1), (cc_name@2, cc_name@2), (CAST(v1.rn AS Decimal128(21, 0))@9, v1_lead.rn - Decimal128(Some(1),20,0)@5)], projection=[i_category@0, i_brand@1, cc_name@2, d_year@3, d_moy@4, sum_sales@5, avg_monthly_sales@6, sum_sales@8, sum_sales@13] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy, sum_sales@5 as sum_sales, avg_monthly_sales@6 as avg_monthly_sales, rn@7 as rn, sum_sales@8 as sum_sales, CAST(rn@7 AS Decimal128(21, 0)) as CAST(v1.rn AS Decimal128(21, 0))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_category@0, i_category@0), (i_brand@1, i_brand@1), (cc_name@2, cc_name@2), (CAST(v1.rn AS Decimal128(21, 0))@8, v1_lag.rn + Decimal128(Some(1),20,0)@5)], projection=[i_category@0, i_brand@1, cc_name@2, d_year@3, d_moy@4, sum_sales@5, avg_monthly_sales@6, rn@7, sum_sales@12] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy, sum(catalog_sales.cs_sales_price)@5 as sum_sales, avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 as avg_monthly_sales, rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 AS Decimal128(21, 0)) as CAST(v1.rn AS Decimal128(21, 0))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@3 = 1999 AND avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 > Some(0),19,6 AND CASE WHEN avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 > Some(0),19,6 THEN abs(sum(catalog_sales.cs_sales_price)@5 - avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6) / avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 END > Some(1000000000),30,10 + │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_year@3 ASC NULLS LAST, d_moy@4 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2], 3), input_partitions=3 + │ BoundedWindowAggExec: wdw=[avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Decimal128(19, 6) }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_year@3 ASC NULLS LAST, d_moy@4 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2, d_year@3], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy], aggr=[sum(catalog_sales.cs_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2, d_year@3, d_moy@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@1 as i_category, i_brand@0 as i_brand, cc_name@5 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy], aggr=[sum(catalog_sales.cs_sales_price)] + │ ProjectionExec: expr=[i_brand@1 as i_brand, i_category@2 as i_category, cs_sales_price@3 as cs_sales_price, d_year@4 as d_year, d_moy@5 as d_moy, cc_name@0 as cc_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(call_center.cc_call_center_sk AS Float64)@2, cs_call_center_sk@2)], projection=[cc_name@1, i_brand@3, i_category@4, cs_sales_price@6, d_year@7, d_moy@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[i_brand@2 as i_brand, i_category@3 as i_category, cs_call_center_sk@4 as cs_call_center_sk, cs_sales_price@5 as cs_sales_price, d_year@0 as d_year, d_moy@1 as d_moy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, cs_sold_date_sk@2)], projection=[d_year@1, d_moy@2, i_brand@4, i_category@5, cs_call_center_sk@7, cs_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@2)], projection=[i_brand@1, i_category@2, cs_sold_date_sk@3, cs_call_center_sk@4, cs_sales_price@6] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, sum(catalog_sales.cs_sales_price)@5 as sum_sales, rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 AS Decimal128(20, 0)) + Some(1),20,0 as v1_lag.rn + Decimal128(Some(1),20,0)] + │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_year@3 ASC NULLS LAST, d_moy@4 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy], aggr=[sum(catalog_sales.cs_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2, d_year@3, d_moy@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@1 as i_category, i_brand@0 as i_brand, cc_name@5 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy], aggr=[sum(catalog_sales.cs_sales_price)] + │ ProjectionExec: expr=[i_brand@1 as i_brand, i_category@2 as i_category, cs_sales_price@3 as cs_sales_price, d_year@4 as d_year, d_moy@5 as d_moy, cc_name@0 as cc_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(call_center.cc_call_center_sk AS Float64)@2, cs_call_center_sk@2)], projection=[cc_name@1, i_brand@3, i_category@4, cs_sales_price@6, d_year@7, d_moy@8] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[i_brand@2 as i_brand, i_category@3 as i_category, cs_call_center_sk@4 as cs_call_center_sk, cs_sales_price@5 as cs_sales_price, d_year@0 as d_year, d_moy@1 as d_moy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, cs_sold_date_sk@2)], projection=[d_year@1, d_moy@2, i_brand@4, i_category@5, cs_call_center_sk@7, cs_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@2)], projection=[i_brand@1, i_category@2, cs_sold_date_sk@3, cs_call_center_sk@4, cs_sales_price@6] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, sum(catalog_sales.cs_sales_price)@5 as sum_sales, rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 AS Decimal128(20, 0)) - Some(1),20,0 as v1_lead.rn - Decimal128(Some(1),20,0)] + │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_year@3 ASC NULLS LAST, d_moy@4 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy], aggr=[sum(catalog_sales.cs_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2, d_year@3, d_moy@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@1 as i_category, i_brand@0 as i_brand, cc_name@5 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy], aggr=[sum(catalog_sales.cs_sales_price)] + │ ProjectionExec: expr=[i_brand@1 as i_brand, i_category@2 as i_category, cs_sales_price@3 as cs_sales_price, d_year@4 as d_year, d_moy@5 as d_moy, cc_name@0 as cc_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(call_center.cc_call_center_sk AS Float64)@2, cs_call_center_sk@2)], projection=[cc_name@1, i_brand@3, i_category@4, cs_sales_price@6, d_year@7, d_moy@8] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[i_brand@2 as i_brand, i_category@3 as i_category, cs_call_center_sk@4 as cs_call_center_sk, cs_sales_price@5 as cs_sales_price, d_year@0 as d_year, d_moy@1 as d_moy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, cs_sold_date_sk@2)], projection=[d_year@1, d_moy@2, i_brand@4, i_category@5, cs_call_center_sk@7, cs_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@2)], projection=[i_brand@1, i_category@2, cs_sold_date_sk@3, cs_call_center_sk@4, cs_sales_price@6] + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[cc_call_center_sk@0 as cc_call_center_sk, cc_name@1 as cc_name, CAST(cc_call_center_sk@0 AS Float64) as CAST(call_center.cc_call_center_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/call_center/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-3.parquet]]}, projection=[cc_call_center_sk, cc_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_moy@2 as d_moy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[cc_call_center_sk@0 as cc_call_center_sk, cc_name@1 as cc_name, CAST(cc_call_center_sk@0 AS Float64) as CAST(call_center.cc_call_center_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/call_center/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-3.parquet]]}, projection=[cc_call_center_sk, cc_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_moy@2 as d_moy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[cc_call_center_sk@0 as cc_call_center_sk, cc_name@1 as cc_name, CAST(cc_call_center_sk@0 AS Float64) as CAST(call_center.cc_call_center_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/call_center/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-3.parquet]]}, projection=[cc_call_center_sk, cc_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_moy@2 as d_moy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_58() -> Result<()> { + let display = test_tpcds_query(58).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [item_id@0 ASC, ss_item_rev@1 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[item_id@0 ASC, ss_item_rev@1 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[item_id@1 as item_id, ss_item_rev@2 as ss_item_rev, ss_item_rev@2 / __common_expr_7@0 * Some(100),20,0 as ss_dev, cs_item_rev@3 as cs_item_rev, cs_item_rev@3 / __common_expr_7@0 * Some(100),20,0 as cs_dev, ws_item_rev@4 as ws_item_rev, ws_item_rev@4 / __common_expr_7@0 * Some(100),20,0 as ws_dev, __common_expr_7@0 as average] + │ ProjectionExec: expr=[(ss_item_rev@2 + cs_item_rev@3 + ws_item_rev@0) / Some(3),20,0 as __common_expr_7, item_id@1 as item_id, ss_item_rev@2 as ss_item_rev, cs_item_rev@3 as cs_item_rev, ws_item_rev@0 as ws_item_rev] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(item_id@0, item_id@0)], filter=CAST(ss_item_rev@0 AS Decimal128(30, 15)) >= CAST(0.9 * CAST(ws_item_rev@2 AS Float64) AS Decimal128(30, 15)) AND CAST(ss_item_rev@0 AS Decimal128(30, 15)) <= CAST(1.1 * CAST(ws_item_rev@2 AS Float64) AS Decimal128(30, 15)) AND CAST(cs_item_rev@1 AS Decimal128(30, 15)) >= CAST(0.9 * CAST(ws_item_rev@2 AS Float64) AS Decimal128(30, 15)) AND CAST(cs_item_rev@1 AS Decimal128(30, 15)) <= CAST(1.1 * CAST(ws_item_rev@2 AS Float64) AS Decimal128(30, 15)) AND CAST(ws_item_rev@2 AS Decimal128(30, 15)) >= CAST(0.9 * CAST(ss_item_rev@0 AS Float64) AS Decimal128(30, 15)) AND CAST(ws_item_rev@2 AS Decimal128(30, 15)) <= CAST(1.1 * CAST(ss_item_rev@0 AS Float64) AS Decimal128(30, 15)) AND CAST(ws_item_rev@2 AS Decimal128(30, 15)) >= CAST(0.9 * CAST(cs_item_rev@1 AS Float64) AS Decimal128(30, 15)) AND CAST(ws_item_rev@2 AS Decimal128(30, 15)) <= CAST(1.1 * CAST(cs_item_rev@1 AS Float64) AS Decimal128(30, 15)), projection=[ws_item_rev@1, item_id@2, ss_item_rev@3, cs_item_rev@4] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[i_item_id@0 as item_id, sum(web_sales.ws_ext_sales_price)@1 as ws_item_rev] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(web_sales.ws_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(web_sales.ws_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(d_date@2, d_date@0)], projection=[ws_ext_sales_price@0, i_item_id@1] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@1] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[item_id@1 as item_id, ss_item_rev@2 as ss_item_rev, cs_item_rev@0 as cs_item_rev] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(item_id@0, item_id@0)], filter=CAST(ss_item_rev@0 AS Decimal128(30, 15)) >= CAST(0.9 * CAST(cs_item_rev@1 AS Float64) AS Decimal128(30, 15)) AND CAST(ss_item_rev@0 AS Decimal128(30, 15)) <= CAST(1.1 * CAST(cs_item_rev@1 AS Float64) AS Decimal128(30, 15)) AND CAST(cs_item_rev@1 AS Decimal128(30, 15)) >= CAST(0.9 * CAST(ss_item_rev@0 AS Float64) AS Decimal128(30, 15)) AND CAST(cs_item_rev@1 AS Decimal128(30, 15)) <= CAST(1.1 * CAST(ss_item_rev@0 AS Float64) AS Decimal128(30, 15)), projection=[cs_item_rev@1, item_id@2, ss_item_rev@3] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[i_item_id@0 as item_id, sum(catalog_sales.cs_ext_sales_price)@1 as cs_item_rev] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(d_date@2, d_date@0)], projection=[cs_ext_sales_price@0, i_item_id@1] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@1] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[i_item_id@0 as item_id, sum(store_sales.ss_ext_sales_price)@1 as ss_item_rev] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(d_date@2, d_date@0)], projection=[ss_ext_sales_price@0, i_item_id@1] + │ CoalescePartitionsExec + │ [Stage 17] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@1] + │ CoalescePartitionsExec + │ [Stage 18] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ ProjectionExec: expr=[ws_ext_sales_price@1 as ws_ext_sales_price, i_item_id@2 as i_item_id, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@0)], projection=[d_date@1, ws_ext_sales_price@4, i_item_id@5] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@2], 6), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_sold_date_sk@0], 6), input_partitions=3 + │ ProjectionExec: expr=[ws_sold_date_sk@1 as ws_sold_date_sk, ws_ext_sales_price@2 as ws_ext_sales_price, i_item_id@0 as i_item_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_item_id@1, ws_sold_date_sk@2, ws_ext_sales_price@4] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@0 = 2000-01-03, projection=[d_week_seq@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@0 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ ProjectionExec: expr=[cs_ext_sales_price@1 as cs_ext_sales_price, i_item_id@2 as i_item_id, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@0)], projection=[d_date@1, cs_ext_sales_price@4, i_item_id@5] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@2], 6), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_sold_date_sk@0], 6), input_partitions=3 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_ext_sales_price@2 as cs_ext_sales_price, i_item_id@0 as i_item_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_item_id@1, cs_sold_date_sk@2, cs_ext_sales_price@4] + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@0 = 2000-01-03, projection=[d_week_seq@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@0 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] + └────────────────────────────────────────────────── + ┌───── Stage 17 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ ProjectionExec: expr=[ss_ext_sales_price@1 as ss_ext_sales_price, i_item_id@2 as i_item_id, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_date@1, ss_ext_sales_price@4, i_item_id@5] + │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@2], 6), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_sold_date_sk@0], 6), input_partitions=3 + │ ProjectionExec: expr=[ss_sold_date_sk@1 as ss_sold_date_sk, ss_ext_sales_price@2 as ss_ext_sales_price, i_item_id@0 as i_item_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_item_id@1, ss_sold_date_sk@2, ss_ext_sales_price@4] + │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 18 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@0 = 2000-01-03, projection=[d_week_seq@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@0 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_59() -> Result<()> { + let display = test_tpcds_query(59).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [s_store_name1@0 ASC, s_store_id1@1 ASC, d_week_seq1@2 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[s_store_name1@0 ASC, s_store_id1@1 ASC, d_week_seq1@2 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[s_store_name1@0 as s_store_name1, s_store_id1@2 as s_store_id1, d_week_seq1@1 as d_week_seq1, sun_sales1@3 / sun_sales2@10 as sun_sales_ratio, mon_sales1@4 / mon_sales2@11 as mon_sales_ratio, tue_sales1@5 / tue_sales2@12 as tue_sales_ratio, wed_sales1@6 / wed_sales2@13 as wed_sales_ratio, thu_sales1@7 / thu_sales2@14 as thu_sales_ratio, fri_sales1@8 / fri_sales2@15 as fri_sales_ratio, sat_sales1@9 / sat_sales2@16 as sat_sales_ratio] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_store_id1@2, s_store_id2@1), (CAST(y.d_week_seq1 AS Int64)@10, x.d_week_seq2 - Int64(52)@9)], projection=[s_store_name1@0, d_week_seq1@1, s_store_id1@2, sun_sales1@3, mon_sales1@4, tue_sales1@5, wed_sales1@6, thu_sales1@7, fri_sales1@8, sat_sales1@9, sun_sales2@13, mon_sales2@14, tue_sales2@15, wed_sales2@16, thu_sales2@17, fri_sales2@18, sat_sales2@19] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[s_store_name@9 as s_store_name1, d_week_seq@0 as d_week_seq1, s_store_id@8 as s_store_id1, sun_sales@1 as sun_sales1, mon_sales@2 as mon_sales1, tue_sales@3 as tue_sales1, wed_sales@4 as wed_sales1, thu_sales@5 as thu_sales1, fri_sales@6 as fri_sales1, sat_sales@7 as sat_sales1, CAST(d_week_seq@0 AS Int64) as CAST(y.d_week_seq1 AS Int64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@0)], projection=[d_week_seq@0, sun_sales@1, mon_sales@2, tue_sales@3, wed_sales@4, thu_sales@5, fri_sales@6, sat_sales@7, s_store_id@8, s_store_name@9] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[d_week_seq@2 as d_week_seq, sun_sales@3 as sun_sales, mon_sales@4 as mon_sales, tue_sales@5 as tue_sales, wed_sales@6 as wed_sales, thu_sales@7 as thu_sales, fri_sales@8 as fri_sales, sat_sales@9 as sat_sales, s_store_id@0 as s_store_id, s_store_name@1 as s_store_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@3, ss_store_sk@1)], projection=[s_store_id@1, s_store_name@2, d_week_seq@4, sun_sales@6, mon_sales@7, tue_sales@8, wed_sales@9, thu_sales@10, fri_sales@11, sat_sales@12] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq, ss_store_sk@1 as ss_store_sk, sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END)@2 as sun_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END)@3 as mon_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END)@4 as tue_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END)@5 as wed_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END)@6 as thu_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END)@7 as fri_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)@8 as sat_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[d_week_seq@0 as d_week_seq, ss_store_sk@1 as ss_store_sk], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_month_seq@0 >= 1212 AND d_month_seq@0 <= 1223, projection=[d_week_seq@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_month_seq, d_week_seq], file_type=parquet, predicate=d_month_seq@0 >= 1212 AND d_month_seq@0 <= 1223 AND DynamicFilter [ empty ], pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1212 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1223, required_guarantees=[] + │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq2, s_store_id@8 as s_store_id2, sun_sales@1 as sun_sales2, mon_sales@2 as mon_sales2, tue_sales@3 as tue_sales2, wed_sales@4 as wed_sales2, thu_sales@5 as thu_sales2, fri_sales@6 as fri_sales2, sat_sales@7 as sat_sales2, CAST(d_week_seq@0 AS Int64) - 52 as x.d_week_seq2 - Int64(52)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@0)], projection=[d_week_seq@0, sun_sales@1, mon_sales@2, tue_sales@3, wed_sales@4, thu_sales@5, fri_sales@6, sat_sales@7, s_store_id@8] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[d_week_seq@1 as d_week_seq, sun_sales@2 as sun_sales, mon_sales@3 as mon_sales, tue_sales@4 as tue_sales, wed_sales@5 as wed_sales, thu_sales@6 as thu_sales, fri_sales@7 as fri_sales, sat_sales@8 as sat_sales, s_store_id@0 as s_store_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, ss_store_sk@1)], projection=[s_store_id@1, d_week_seq@3, sun_sales@5, mon_sales@6, tue_sales@7, wed_sales@8, thu_sales@9, fri_sales@10, sat_sales@11] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq, ss_store_sk@1 as ss_store_sk, sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END)@2 as sun_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END)@3 as mon_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END)@4 as tue_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END)@5 as wed_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END)@6 as thu_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END)@7 as fri_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)@8 as sat_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[d_week_seq@0 as d_week_seq, ss_store_sk@1 as ss_store_sk], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_month_seq@0 >= 1224 AND d_month_seq@0 <= 1235, projection=[d_week_seq@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_month_seq, d_week_seq], file_type=parquet, predicate=d_month_seq@0 >= 1224 AND d_month_seq@0 <= 1235 AND DynamicFilter [ empty ], pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1224 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1235, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_id@1 as s_store_id, s_store_name@2 as s_store_name, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id, s_store_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([d_week_seq@0, ss_store_sk@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_week_seq@2 as d_week_seq, ss_store_sk@0 as ss_store_sk], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] + │ ProjectionExec: expr=[ss_store_sk@2 as ss_store_sk, ss_sales_price@3 as ss_sales_price, d_week_seq@0 as d_week_seq, d_day_name@1 as d_day_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@0)], projection=[d_week_seq@1, d_day_name@2, ss_store_sk@5, ss_sales_price@6] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 6), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_week_seq@1 as d_week_seq, d_day_name@2 as d_day_name, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_sold_date_sk@0], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_id@1 as s_store_id, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([d_week_seq@0, ss_store_sk@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_week_seq@2 as d_week_seq, ss_store_sk@0 as ss_store_sk], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] + │ ProjectionExec: expr=[ss_store_sk@2 as ss_store_sk, ss_sales_price@3 as ss_sales_price, d_week_seq@0 as d_week_seq, d_day_name@1 as d_day_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@0)], projection=[d_week_seq@1, d_day_name@2, ss_store_sk@5, ss_sales_price@6] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 6), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_week_seq@1 as d_week_seq, d_day_name@2 as d_day_name, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_sold_date_sk@0], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_60() -> Result<()> { + let display = test_tpcds_query(60).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST, total_sales@1 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST, total_sales@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(tmp1.total_sales)@1 as total_sales] + │ AggregateExec: mode=SinglePartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(tmp1.total_sales)] + │ InterleaveExec + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(store_sales.ss_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(i_item_id@1, i_item_id@0)] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_ext_sales_price@1, i_item_id@3] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@1)], projection=[ss_item_sk@0, ss_ext_sales_price@2] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_addr_sk@4, ss_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@1 = Some(-500),4,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_category@1 = Music, projection=[i_item_id@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@1 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(catalog_sales.cs_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(i_item_id@1, i_item_id@0)] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, i_item_sk@0)], projection=[cs_ext_sales_price@1, i_item_id@3] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_bill_addr_sk@0, CAST(customer_address.ca_address_sk AS Float64)@1)], projection=[cs_item_sk@1, cs_ext_sales_price@2] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_bill_addr_sk@3, cs_item_sk@4, cs_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@1 = Some(-500),4,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_category@1 = Music, projection=[i_item_id@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@1 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(web_sales.ws_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(web_sales.ws_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(web_sales.ws_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(i_item_id@1, i_item_id@0)] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@0, i_item_sk@0)], projection=[ws_ext_sales_price@1, i_item_id@3] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_bill_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@1)], projection=[ws_item_sk@0, ws_ext_sales_price@2] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_bill_addr_sk@4, ws_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@1 = Some(-500),4,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_category@1 = Music, projection=[i_item_id@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@1 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 9, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 1998 AND d_moy@2 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 9, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 1998 AND d_moy@2 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 9, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 1998 AND d_moy@2 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_61() -> Result<()> { + let display = test_tpcds_query(61).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortExec: TopK(fetch=100), expr=[total@1 ASC NULLS LAST], preserve_partitioning=[false] + │ ProjectionExec: expr=[promotions@0 as promotions, total@1 as total, CAST(promotions@0 AS Decimal128(15, 4)) / CAST(total@1 AS Decimal128(15, 4)) * Some(100),20,0 as promotional_sales.promotions / all_sales.total * Int64(100)] + │ CrossJoinExec + │ ProjectionExec: expr=[sum(store_sales.ss_ext_sales_price)@0 as promotions] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_ext_sales_price@1] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@2, ca_address_sk@0)], projection=[ss_item_sk@0, ss_ext_sales_price@1] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@1, CAST(customer.c_customer_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_ext_sales_price@2, c_current_addr_sk@4] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_sold_date_sk@0, CAST(date_dim.d_date_sk AS Float64)@1)], projection=[ss_item_sk@1, ss_customer_sk@2, ss_ext_sales_price@3] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_promo_sk@3, CAST(promotion.p_promo_sk AS Float64)@1)], projection=[ss_sold_date_sk@0, ss_item_sk@1, ss_customer_sk@2, ss_ext_sales_price@4] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@3)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_customer_sk@4, ss_promo_sk@6, ss_ext_sales_price@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_promo_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: p_channel_dmail@1 = Y OR p_channel_email@2 = Y OR p_channel_tv@3 = Y, projection=[p_promo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, p_channel_dmail, p_channel_email, p_channel_tv], file_type=parquet, predicate=p_channel_dmail@1 = Y OR p_channel_email@2 = Y OR p_channel_tv@3 = Y, pruning_predicate=p_channel_dmail_null_count@2 != row_count@3 AND p_channel_dmail_min@0 <= Y AND Y <= p_channel_dmail_max@1 OR p_channel_email_null_count@6 != row_count@3 AND p_channel_email_min@4 <= Y AND Y <= p_channel_email_max@5 OR p_channel_tv_null_count@9 != row_count@3 AND p_channel_tv_min@7 <= Y AND Y <= p_channel_tv_max@8, required_guarantees=[] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 11, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 1998 AND d_moy@2 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@1 = Some(-500),4,2 AND DynamicFilter [ empty ], pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_category@1 = Jewelry, projection=[i_item_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category], file_type=parquet, predicate=i_category@1 = Jewelry AND DynamicFilter [ empty ], pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1, required_guarantees=[i_category in (Jewelry)] + │ ProjectionExec: expr=[sum(store_sales.ss_ext_sales_price)@0 as total] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_ext_sales_price@1] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@2, ca_address_sk@0)], projection=[ss_item_sk@0, ss_ext_sales_price@1] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@1, CAST(customer.c_customer_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_ext_sales_price@2, c_current_addr_sk@4] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_sold_date_sk@0, CAST(date_dim.d_date_sk AS Float64)@1)], projection=[ss_item_sk@1, ss_customer_sk@2, ss_ext_sales_price@3] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@3)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_customer_sk@4, ss_ext_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 11, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 1998 AND d_moy@2 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@1 = Some(-500),4,2 AND DynamicFilter [ empty ], pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_category@1 = Jewelry, projection=[i_item_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category], file_type=parquet, predicate=i_category@1 = Jewelry AND DynamicFilter [ empty ], pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1, required_guarantees=[i_category in (Jewelry)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_gmt_offset@1 = Some(-500),3,2, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@1 = Some(-500),3,2, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= Some(-500),3,2 AND Some(-500),3,2 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (Some(-500),3,2)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_gmt_offset@1 = Some(-500),3,2, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@1 = Some(-500),3,2, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= Some(-500),3,2 AND Some(-500),3,2 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (Some(-500),3,2)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_62() -> Result<()> { + let display = test_tpcds_query(62).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [w_substr@0 ASC, sm_type@1 ASC, web_name@2 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[w_substr@0 ASC, sm_type@1 ASC, web_name@2 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[w_substr@0 as w_substr, sm_type@1 as sm_type, web_name@2 as web_name, sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END)@3 as 30 days, sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(30) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END)@4 as 31-60 days, sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(60) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END)@5 as 61-90 days, sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(90) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END)@6 as 91-120 days, sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)@7 as >120 days] + │ AggregateExec: mode=FinalPartitioned, gby=[w_substr@0 as w_substr, sm_type@1 as sm_type, web_name@2 as web_name], aggr=[sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(30) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(60) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(90) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([w_substr@0, sm_type@1, web_name@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[w_substr@1 as w_substr, sm_type@2 as sm_type, web_name@3 as web_name], aggr=[sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(30) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(60) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(90) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)] + │ ProjectionExec: expr=[ws_ship_date_sk@1 - ws_sold_date_sk@0 as __common_expr_1, w_substr@2 as w_substr, sm_type@3 as sm_type, web_name@4 as web_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_ship_date_sk@1, CAST(date_dim.d_date_sk AS Float64)@1)], projection=[ws_sold_date_sk@0, ws_ship_date_sk@1, w_substr@2, sm_type@3, web_name@4] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_web_site_sk@2, CAST(web_site.web_site_sk AS Float64)@2)], projection=[ws_sold_date_sk@0, ws_ship_date_sk@1, w_substr@3, sm_type@4, web_name@6] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_ship_mode_sk@3, CAST(ship_mode.sm_ship_mode_sk AS Float64)@2)], projection=[ws_sold_date_sk@0, ws_ship_date_sk@1, ws_web_site_sk@2, w_substr@4, sm_type@6] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ws_sold_date_sk@1 as ws_sold_date_sk, ws_ship_date_sk@2 as ws_ship_date_sk, ws_web_site_sk@3 as ws_web_site_sk, ws_ship_mode_sk@4 as ws_ship_mode_sk, w_substr@0 as w_substr] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(sq1.w_warehouse_sk AS Float64)@2, ws_warehouse_sk@4)], projection=[w_substr@0, ws_sold_date_sk@3, ws_ship_date_sk@4, ws_web_site_sk@5, ws_ship_mode_sk@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_ship_date_sk, ws_web_site_sk, ws_ship_mode_sk, ws_warehouse_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[sm_ship_mode_sk@0 as sm_ship_mode_sk, sm_type@1 as sm_type, CAST(sm_ship_mode_sk@0 AS Float64) as CAST(ship_mode.sm_ship_mode_sk AS Float64)] + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/ship_mode/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/ship_mode/part-3.parquet]]}, projection=[sm_ship_mode_sk, sm_type], file_type=parquet + │ ProjectionExec: expr=[web_site_sk@0 as web_site_sk, web_name@1 as web_name, CAST(web_site_sk@0 AS Float64) as CAST(web_site.web_site_sk AS Float64)] + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/web_site/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/web_site/part-3.parquet]]}, projection=[web_site_sk, web_name], file_type=parquet + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[substr(w_warehouse_name@1, 1, 20) as w_substr, w_warehouse_sk@0 as w_warehouse_sk, CAST(w_warehouse_sk@0 AS Float64) as CAST(sq1.w_warehouse_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet]]}, projection=[w_warehouse_sk, w_warehouse_name], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_63() -> Result<()> { + let display = test_tpcds_query(63).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_manager_id@0 ASC NULLS LAST, avg_monthly_sales@2 ASC NULLS LAST, sum_sales@1 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_manager_id@0 ASC NULLS LAST, avg_monthly_sales@2 ASC NULLS LAST, sum_sales@1 ASC NULLS LAST], preserve_partitioning=[true], sort_prefix=[i_manager_id@0 ASC NULLS LAST, avg_monthly_sales@2 ASC NULLS LAST] + │ ProjectionExec: expr=[i_manager_id@0 as i_manager_id, sum(store_sales.ss_sales_price)@1 as sum_sales, avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 as avg_monthly_sales] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CASE WHEN avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 > Some(0),19,6 THEN abs(sum(store_sales.ss_sales_price)@1 - avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2) / avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 END > Some(1000000000),30,10 + │ WindowAggExec: wdw=[avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Decimal128(19, 6), nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + │ SortExec: expr=[i_manager_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_manager_id@0], 3), input_partitions=3 + │ ProjectionExec: expr=[i_manager_id@0 as i_manager_id, sum(store_sales.ss_sales_price)@2 as sum(store_sales.ss_sales_price)] + │ AggregateExec: mode=FinalPartitioned, gby=[i_manager_id@0 as i_manager_id, d_moy@1 as d_moy], aggr=[sum(store_sales.ss_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_manager_id@0, d_moy@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_manager_id@0 as i_manager_id, d_moy@2 as d_moy], aggr=[sum(store_sales.ss_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@1)], projection=[i_manager_id@2, ss_sales_price@4, d_moy@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[i_manager_id@1 as i_manager_id, ss_store_sk@2 as ss_store_sk, ss_sales_price@3 as ss_sales_price, d_moy@0 as d_moy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@1)], projection=[d_moy@1, i_manager_id@3, ss_store_sk@5, ss_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_manager_id@1, ss_sold_date_sk@2, ss_store_sk@4, ss_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_moy@1 as d_moy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_month_seq@1 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), projection=[d_date_sk@0, d_moy@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq, d_moy], file_type=parquet, predicate=d_month_seq@1 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), pruning_predicate=d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1200 AND 1200 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1201 AND 1201 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1202 AND 1202 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1203 AND 1203 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1204 AND 1204 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1205 AND 1205 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1206 AND 1206 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1207 AND 1207 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1208 AND 1208 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1209 AND 1209 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1210 AND 1210 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1211 AND 1211 <= d_month_seq_max@1, required_guarantees=[d_month_seq in (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: (i_category@3 = Books OR i_category@3 = Children OR i_category@3 = Electronics) AND i_class@2 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@1 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@3 = Women OR i_category@3 = Music OR i_category@3 = Men) AND i_class@2 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@1 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), projection=[i_item_sk@0, i_manager_id@4] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_manager_id], file_type=parquet, predicate=(i_category@3 = Books OR i_category@3 = Children OR i_category@3 = Electronics) AND i_class@2 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@1 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@3 = Women OR i_category@3 = Music OR i_category@3 = Men) AND i_class@2 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@1 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Children AND Children <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= personal AND personal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= portable AND portable <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= reference AND reference <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= self-help AND self-help <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #14 AND scholaramalgamalg #14 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #7 AND scholaramalgamalg #7 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiunivamalg #9 AND exportiunivamalg #9 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #9 AND scholaramalgamalg #9 <= i_brand_max@8) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= accessories AND accessories <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= classical AND classical <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= fragrances AND fragrances <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= pants AND pants <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= amalgimporto #1 AND amalgimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= edu packscholar #1 AND edu packscholar #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiimporto #1 AND exportiimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= importoamalg #1 AND importoamalg #1 <= i_brand_max@8), required_guarantees=[i_brand in (amalgimporto #1, edu packscholar #1, exportiimporto #1, exportiunivamalg #9, importoamalg #1, scholaramalgamalg #14, scholaramalgamalg #7, scholaramalgamalg #9), i_class in (accessories, classical, fragrances, pants, personal, portable, reference, self-help)] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_64() -> Result<()> { + let display = test_tpcds_query(64).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[product_name@0 as product_name, store_name@1 as store_name, store_zip@2 as store_zip, b_street_number@3 as b_street_number, b_street_name@4 as b_street_name, b_city@5 as b_city, b_zip@6 as b_zip, c_street_number@7 as c_street_number, c_street_name@8 as c_street_name, c_city@9 as c_city, c_zip@10 as c_zip, cs1syear@11 as cs1syear, cs1cnt@12 as cs1cnt, s11@13 as s11, s21@14 as s21, s31@15 as s31, s12@16 as s12, s22@17 as s22, s32@18 as s32, syear@19 as syear, cnt@20 as cnt] + │ SortPreservingMergeExec: [product_name@0 ASC NULLS LAST, store_name@1 ASC NULLS LAST, cnt@20 ASC NULLS LAST, s1@21 ASC NULLS LAST, s1@22 ASC NULLS LAST] + │ SortExec: expr=[product_name@0 ASC NULLS LAST, store_name@1 ASC NULLS LAST, cnt@20 ASC NULLS LAST, s11@13 ASC NULLS LAST, s12@16 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[product_name@0 as product_name, store_name@1 as store_name, store_zip@2 as store_zip, b_street_number@3 as b_street_number, b_street_name@4 as b_street_name, b_city@5 as b_city, b_zip@6 as b_zip, c_street_number@7 as c_street_number, c_street_name@8 as c_street_name, c_city@9 as c_city, c_zip@10 as c_zip, syear@11 as cs1syear, cnt@12 as cs1cnt, s1@13 as s11, s2@14 as s21, s3@15 as s31, s1@18 as s12, s2@19 as s22, s3@20 as s32, syear@16 as syear, cnt@17 as cnt, s1@13 as s1, s1@18 as s1] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(item_sk@1, item_sk@0), (store_name@2, store_name@1), (store_zip@3, store_zip@2)], filter=cnt@1 <= cnt@0, projection=[product_name@0, store_name@2, store_zip@3, b_street_number@4, b_street_name@5, b_city@6, b_zip@7, c_street_number@8, c_street_name@9, c_city@10, c_zip@11, syear@12, cnt@13, s1@14, s2@15, s3@16, syear@20, cnt@21, s1@22, s2@23, s3@24] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[i_product_name@0 as product_name, i_item_sk@1 as item_sk, s_store_name@2 as store_name, s_zip@3 as store_zip, ca_street_number@4 as b_street_number, ca_street_name@5 as b_street_name, ca_city@6 as b_city, ca_zip@7 as b_zip, ca_street_number@8 as c_street_number, ca_street_name@9 as c_street_name, ca_city@10 as c_city, ca_zip@11 as c_zip, d_year@12 as syear, count(Int64(1))@15 as cnt, sum(store_sales.ss_wholesale_cost)@16 as s1, sum(store_sales.ss_list_price)@17 as s2, sum(store_sales.ss_coupon_amt)@18 as s3] + │ AggregateExec: mode=FinalPartitioned, gby=[i_product_name@0 as i_product_name, i_item_sk@1 as i_item_sk, s_store_name@2 as s_store_name, s_zip@3 as s_zip, ca_street_number@4 as ca_street_number, ca_street_name@5 as ca_street_name, ca_city@6 as ca_city, ca_zip@7 as ca_zip, ca_street_number@8 as ca_street_number, ca_street_name@9 as ca_street_name, ca_city@10 as ca_city, ca_zip@11 as ca_zip, d_year@12 as d_year, d_year@13 as d_year, d_year@14 as d_year], aggr=[count(Int64(1)), sum(store_sales.ss_wholesale_cost), sum(store_sales.ss_list_price), sum(store_sales.ss_coupon_amt)], ordering_mode=PartiallySorted([12]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_product_name@0, i_item_sk@1, s_store_name@2, s_zip@3, ca_street_number@4, ca_street_name@5, ca_city@6, ca_zip@7, ca_street_number@8, ca_street_name@9, ca_city@10, ca_zip@11, d_year@12, d_year@13, d_year@14], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_product_name@17 as i_product_name, i_item_sk@16 as i_item_sk, s_store_name@6 as s_store_name, s_zip@7 as s_zip, ca_street_number@8 as ca_street_number, ca_street_name@9 as ca_street_name, ca_city@10 as ca_city, ca_zip@11 as ca_zip, ca_street_number@12 as ca_street_number, ca_street_name@13 as ca_street_name, ca_city@14 as ca_city, ca_zip@15 as ca_zip, d_year@3 as d_year, d_year@4 as d_year, d_year@5 as d_year], aggr=[count(Int64(1)), sum(store_sales.ss_wholesale_cost), sum(store_sales.ss_list_price), sum(store_sales.ss_coupon_amt)], ordering_mode=PartiallySorted([12]) + │ ProjectionExec: expr=[ss_wholesale_cost@0 as ss_wholesale_cost, ss_list_price@1 as ss_list_price, ss_coupon_amt@2 as ss_coupon_amt, d_year@3 as d_year, d_year@6 as d_year, d_year@7 as d_year, s_store_name@4 as s_store_name, s_zip@5 as s_zip, ca_street_number@8 as ca_street_number, ca_street_name@9 as ca_street_name, ca_city@10 as ca_city, ca_zip@11 as ca_zip, ca_street_number@12 as ca_street_number, ca_street_name@13 as ca_street_name, ca_city@14 as ca_city, ca_zip@15 as ca_zip, i_item_sk@16 as i_item_sk, i_product_name@17 as i_product_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_wholesale_cost@1, ss_list_price@2, ss_coupon_amt@3, d_year@4, s_store_name@5, s_zip@6, d_year@7, d_year@8, ca_street_number@9, ca_street_name@10, ca_city@11, ca_zip@12, ca_street_number@13, ca_street_name@14, ca_city@15, ca_zip@16, i_item_sk@17, i_product_name@18] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(hd_income_band_sk@9, ib_income_band_sk@0)], projection=[ss_item_sk@0, ss_wholesale_cost@1, ss_list_price@2, ss_coupon_amt@3, d_year@4, s_store_name@5, s_zip@6, d_year@7, d_year@8, ca_street_number@10, ca_street_name@11, ca_city@12, ca_zip@13, ca_street_number@14, ca_street_name@15, ca_city@16, ca_zip@17] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ib_income_band_sk@0, hd_income_band_sk@9)], projection=[ss_item_sk@1, ss_wholesale_cost@2, ss_list_price@3, ss_coupon_amt@4, d_year@5, s_store_name@6, s_zip@7, d_year@8, d_year@9, hd_income_band_sk@11, ca_street_number@12, ca_street_name@13, ca_city@14, ca_zip@15, ca_street_number@16, ca_street_name@17, ca_city@18, ca_zip@19] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet]]}, projection=[ib_income_band_sk], file_type=parquet + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@7, ca_address_sk@0)], projection=[ss_item_sk@0, ss_wholesale_cost@1, ss_list_price@2, ss_coupon_amt@3, d_year@4, s_store_name@5, s_zip@6, d_year@8, d_year@9, hd_income_band_sk@10, hd_income_band_sk@11, ca_street_number@12, ca_street_name@13, ca_city@14, ca_zip@15, ca_street_number@17, ca_street_name@18, ca_city@19, ca_zip@20] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@1, CAST(ad1.ca_address_sk AS Float64)@5)], projection=[ss_item_sk@0, ss_wholesale_cost@2, ss_list_price@3, ss_coupon_amt@4, d_year@5, s_store_name@6, s_zip@7, c_current_addr_sk@8, d_year@9, d_year@10, hd_income_band_sk@11, hd_income_band_sk@12, ca_street_number@14, ca_street_name@15, ca_city@16, ca_zip@17] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_hdemo_sk@8, CAST(hd2.hd_demo_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_addr_sk@1, ss_wholesale_cost@2, ss_list_price@3, ss_coupon_amt@4, d_year@5, s_store_name@6, s_zip@7, c_current_addr_sk@9, d_year@10, d_year@11, hd_income_band_sk@12, hd_income_band_sk@14] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_hdemo_sk@1, CAST(hd1.hd_demo_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_addr_sk@2, ss_wholesale_cost@3, ss_list_price@4, ss_coupon_amt@5, d_year@6, s_store_name@7, s_zip@8, c_current_hdemo_sk@9, c_current_addr_sk@10, d_year@11, d_year@12, hd_income_band_sk@14] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(promotion.p_promo_sk AS Float64)@1, ss_promo_sk@3)], projection=[ss_item_sk@2, ss_hdemo_sk@3, ss_addr_sk@4, ss_wholesale_cost@6, ss_list_price@7, ss_coupon_amt@8, d_year@9, s_store_name@10, s_zip@11, c_current_hdemo_sk@12, c_current_addr_sk@13, d_year@14, d_year@15] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@10, CAST(cd2.cd_demo_sk AS Float64)@2)], filter=cd_marital_status@1 != cd_marital_status@0, projection=[ss_item_sk@0, ss_hdemo_sk@1, ss_addr_sk@2, ss_promo_sk@3, ss_wholesale_cost@4, ss_list_price@5, ss_coupon_amt@6, d_year@7, s_store_name@8, s_zip@9, c_current_hdemo_sk@11, c_current_addr_sk@12, d_year@13, d_year@14] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_cdemo_sk@1, CAST(cd1.cd_demo_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_hdemo_sk@2, ss_addr_sk@3, ss_promo_sk@4, ss_wholesale_cost@5, ss_list_price@6, ss_coupon_amt@7, d_year@8, s_store_name@9, s_zip@10, c_current_cdemo_sk@11, c_current_hdemo_sk@12, c_current_addr_sk@13, d_year@14, d_year@15, cd_marital_status@17] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_first_shipto_date_sk@14, CAST(d3.d_date_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_cdemo_sk@1, ss_hdemo_sk@2, ss_addr_sk@3, ss_promo_sk@4, ss_wholesale_cost@5, ss_list_price@6, ss_coupon_amt@7, d_year@8, s_store_name@9, s_zip@10, c_current_cdemo_sk@11, c_current_hdemo_sk@12, c_current_addr_sk@13, d_year@15, d_year@17] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_first_sales_date_sk@15, CAST(d2.d_date_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_cdemo_sk@1, ss_hdemo_sk@2, ss_addr_sk@3, ss_promo_sk@4, ss_wholesale_cost@5, ss_list_price@6, ss_coupon_amt@7, d_year@8, s_store_name@9, s_zip@10, c_current_cdemo_sk@11, c_current_hdemo_sk@12, c_current_addr_sk@13, c_first_shipto_date_sk@14, d_year@17] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@1, CAST(customer.c_customer_sk AS Float64)@6)], projection=[ss_item_sk@0, ss_cdemo_sk@2, ss_hdemo_sk@3, ss_addr_sk@4, ss_promo_sk@5, ss_wholesale_cost@6, ss_list_price@7, ss_coupon_amt@8, d_year@9, s_store_name@10, s_zip@11, c_current_cdemo_sk@13, c_current_hdemo_sk@14, c_current_addr_sk@15, c_first_shipto_date_sk@16, c_first_sales_date_sk@17] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_item_sk@2 as ss_item_sk, ss_customer_sk@3 as ss_customer_sk, ss_cdemo_sk@4 as ss_cdemo_sk, ss_hdemo_sk@5 as ss_hdemo_sk, ss_addr_sk@6 as ss_addr_sk, ss_promo_sk@7 as ss_promo_sk, ss_wholesale_cost@8 as ss_wholesale_cost, ss_list_price@9 as ss_list_price, ss_coupon_amt@10 as ss_coupon_amt, d_year@11 as d_year, s_store_name@0 as s_store_name, s_zip@1 as s_zip] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@3, ss_store_sk@5)], projection=[s_store_name@1, s_zip@2, ss_item_sk@4, ss_customer_sk@5, ss_cdemo_sk@6, ss_hdemo_sk@7, ss_addr_sk@8, ss_promo_sk@10, ss_wholesale_cost@11, ss_list_price@12, ss_coupon_amt@13, d_year@14] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_customer_sk@2 as ss_customer_sk, ss_cdemo_sk@3 as ss_cdemo_sk, ss_hdemo_sk@4 as ss_hdemo_sk, ss_addr_sk@5 as ss_addr_sk, ss_store_sk@6 as ss_store_sk, ss_promo_sk@7 as ss_promo_sk, ss_wholesale_cost@8 as ss_wholesale_cost, ss_list_price@9 as ss_list_price, ss_coupon_amt@10 as ss_coupon_amt, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_customer_sk@5, ss_cdemo_sk@6, ss_hdemo_sk@7, ss_addr_sk@8, ss_store_sk@9, ss_promo_sk@10, ss_wholesale_cost@11, ss_list_price@12, ss_coupon_amt@13] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, ss_item_sk@1)], projection=[ss_sold_date_sk@1, ss_item_sk@2, ss_customer_sk@3, ss_cdemo_sk@4, ss_hdemo_sk@5, ss_addr_sk@6, ss_store_sk@7, ss_promo_sk@8, ss_wholesale_cost@9, ss_list_price@10, ss_coupon_amt@11] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CAST(sum(catalog_sales.cs_ext_list_price)@1 AS Decimal128(38, 2)) > Some(2),20,0 * sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)@2, projection=[cs_item_sk@0] + │ AggregateExec: mode=FinalPartitioned, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(catalog_sales.cs_ext_list_price), sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_item_sk@0, ss_item_sk@1), (sr_ticket_number@1, ss_ticket_number@8)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_customer_sk@4, ss_cdemo_sk@5, ss_hdemo_sk@6, ss_addr_sk@7, ss_store_sk@8, ss_promo_sk@9, ss_wholesale_cost@11, ss_list_price@12, ss_coupon_amt@13] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_cdemo_sk@1 as c_current_cdemo_sk, c_current_hdemo_sk@2 as c_current_hdemo_sk, c_current_addr_sk@3 as c_current_addr_sk, c_first_shipto_date_sk@4 as c_first_shipto_date_sk, c_first_sales_date_sk@5 as c_first_sales_date_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, cd_marital_status@1 as cd_marital_status, CAST(cd_demo_sk@0 AS Float64) as CAST(cd1.cd_demo_sk AS Float64)] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, cd_marital_status@1 as cd_marital_status, CAST(cd_demo_sk@0 AS Float64) as CAST(cd2.cd_demo_sk AS Float64)] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, hd_income_band_sk@1 as hd_income_band_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(hd1.hd_demo_sk AS Float64)] + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, hd_income_band_sk@1 as hd_income_band_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(hd2.hd_demo_sk AS Float64)] + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_street_number@1 as ca_street_number, ca_street_name@2 as ca_street_name, ca_city@3 as ca_city, ca_zip@4 as ca_zip, CAST(ca_address_sk@0 AS Float64) as CAST(ad1.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet]]}, projection=[ib_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_color@2 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@1 >= Some(6500),4,2 AND i_current_price@1 <= Some(7400),4,2, projection=[i_item_sk@0, i_product_name@3] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price, i_color, i_product_name], file_type=parquet, predicate=i_color@2 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@1 >= Some(6500),4,2 AND i_current_price@1 <= Some(7400),4,2 AND DynamicFilter [ empty ], pruning_predicate=(i_color_null_count@2 != row_count@3 AND i_color_min@0 <= purple AND purple <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burlywood AND burlywood <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= indian AND indian <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= spring AND spring <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= floral AND floral <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= medium AND medium <= i_color_max@1) AND i_current_price_null_count@5 != row_count@3 AND i_current_price_max@4 >= Some(6500),4,2 AND i_current_price_null_count@5 != row_count@3 AND i_current_price_min@6 <= Some(7400),4,2, required_guarantees=[i_color in (burlywood, floral, indian, medium, purple, spring)] + │ ProjectionExec: expr=[i_item_sk@1 as item_sk, s_store_name@2 as store_name, s_zip@3 as store_zip, d_year@12 as syear, count(Int64(1))@15 as cnt, sum(store_sales.ss_wholesale_cost)@16 as s1, sum(store_sales.ss_list_price)@17 as s2, sum(store_sales.ss_coupon_amt)@18 as s3] + │ AggregateExec: mode=FinalPartitioned, gby=[i_product_name@0 as i_product_name, i_item_sk@1 as i_item_sk, s_store_name@2 as s_store_name, s_zip@3 as s_zip, ca_street_number@4 as ca_street_number, ca_street_name@5 as ca_street_name, ca_city@6 as ca_city, ca_zip@7 as ca_zip, ca_street_number@8 as ca_street_number, ca_street_name@9 as ca_street_name, ca_city@10 as ca_city, ca_zip@11 as ca_zip, d_year@12 as d_year, d_year@13 as d_year, d_year@14 as d_year], aggr=[count(Int64(1)), sum(store_sales.ss_wholesale_cost), sum(store_sales.ss_list_price), sum(store_sales.ss_coupon_amt)], ordering_mode=PartiallySorted([12]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_product_name@0, i_item_sk@1, s_store_name@2, s_zip@3, ca_street_number@4, ca_street_name@5, ca_city@6, ca_zip@7, ca_street_number@8, ca_street_name@9, ca_city@10, ca_zip@11, d_year@12, d_year@13, d_year@14], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_product_name@17 as i_product_name, i_item_sk@16 as i_item_sk, s_store_name@6 as s_store_name, s_zip@7 as s_zip, ca_street_number@8 as ca_street_number, ca_street_name@9 as ca_street_name, ca_city@10 as ca_city, ca_zip@11 as ca_zip, ca_street_number@12 as ca_street_number, ca_street_name@13 as ca_street_name, ca_city@14 as ca_city, ca_zip@15 as ca_zip, d_year@3 as d_year, d_year@4 as d_year, d_year@5 as d_year], aggr=[count(Int64(1)), sum(store_sales.ss_wholesale_cost), sum(store_sales.ss_list_price), sum(store_sales.ss_coupon_amt)], ordering_mode=PartiallySorted([12]) + │ ProjectionExec: expr=[ss_wholesale_cost@0 as ss_wholesale_cost, ss_list_price@1 as ss_list_price, ss_coupon_amt@2 as ss_coupon_amt, d_year@3 as d_year, d_year@6 as d_year, d_year@7 as d_year, s_store_name@4 as s_store_name, s_zip@5 as s_zip, ca_street_number@8 as ca_street_number, ca_street_name@9 as ca_street_name, ca_city@10 as ca_city, ca_zip@11 as ca_zip, ca_street_number@12 as ca_street_number, ca_street_name@13 as ca_street_name, ca_city@14 as ca_city, ca_zip@15 as ca_zip, i_item_sk@16 as i_item_sk, i_product_name@17 as i_product_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_wholesale_cost@1, ss_list_price@2, ss_coupon_amt@3, d_year@4, s_store_name@5, s_zip@6, d_year@7, d_year@8, ca_street_number@9, ca_street_name@10, ca_city@11, ca_zip@12, ca_street_number@13, ca_street_name@14, ca_city@15, ca_zip@16, i_item_sk@17, i_product_name@18] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(hd_income_band_sk@9, ib_income_band_sk@0)], projection=[ss_item_sk@0, ss_wholesale_cost@1, ss_list_price@2, ss_coupon_amt@3, d_year@4, s_store_name@5, s_zip@6, d_year@7, d_year@8, ca_street_number@10, ca_street_name@11, ca_city@12, ca_zip@13, ca_street_number@14, ca_street_name@15, ca_city@16, ca_zip@17] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ib_income_band_sk@0, hd_income_band_sk@9)], projection=[ss_item_sk@1, ss_wholesale_cost@2, ss_list_price@3, ss_coupon_amt@4, d_year@5, s_store_name@6, s_zip@7, d_year@8, d_year@9, hd_income_band_sk@11, ca_street_number@12, ca_street_name@13, ca_city@14, ca_zip@15, ca_street_number@16, ca_street_name@17, ca_city@18, ca_zip@19] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet]]}, projection=[ib_income_band_sk], file_type=parquet + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@7, ca_address_sk@0)], projection=[ss_item_sk@0, ss_wholesale_cost@1, ss_list_price@2, ss_coupon_amt@3, d_year@4, s_store_name@5, s_zip@6, d_year@8, d_year@9, hd_income_band_sk@10, hd_income_band_sk@11, ca_street_number@12, ca_street_name@13, ca_city@14, ca_zip@15, ca_street_number@17, ca_street_name@18, ca_city@19, ca_zip@20] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@1, CAST(ad1.ca_address_sk AS Float64)@5)], projection=[ss_item_sk@0, ss_wholesale_cost@2, ss_list_price@3, ss_coupon_amt@4, d_year@5, s_store_name@6, s_zip@7, c_current_addr_sk@8, d_year@9, d_year@10, hd_income_band_sk@11, hd_income_band_sk@12, ca_street_number@14, ca_street_name@15, ca_city@16, ca_zip@17] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_hdemo_sk@8, CAST(hd2.hd_demo_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_addr_sk@1, ss_wholesale_cost@2, ss_list_price@3, ss_coupon_amt@4, d_year@5, s_store_name@6, s_zip@7, c_current_addr_sk@9, d_year@10, d_year@11, hd_income_band_sk@12, hd_income_band_sk@14] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_hdemo_sk@1, CAST(hd1.hd_demo_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_addr_sk@2, ss_wholesale_cost@3, ss_list_price@4, ss_coupon_amt@5, d_year@6, s_store_name@7, s_zip@8, c_current_hdemo_sk@9, c_current_addr_sk@10, d_year@11, d_year@12, hd_income_band_sk@14] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(promotion.p_promo_sk AS Float64)@1, ss_promo_sk@3)], projection=[ss_item_sk@2, ss_hdemo_sk@3, ss_addr_sk@4, ss_wholesale_cost@6, ss_list_price@7, ss_coupon_amt@8, d_year@9, s_store_name@10, s_zip@11, c_current_hdemo_sk@12, c_current_addr_sk@13, d_year@14, d_year@15] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@10, CAST(cd2.cd_demo_sk AS Float64)@2)], filter=cd_marital_status@1 != cd_marital_status@0, projection=[ss_item_sk@0, ss_hdemo_sk@1, ss_addr_sk@2, ss_promo_sk@3, ss_wholesale_cost@4, ss_list_price@5, ss_coupon_amt@6, d_year@7, s_store_name@8, s_zip@9, c_current_hdemo_sk@11, c_current_addr_sk@12, d_year@13, d_year@14] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_cdemo_sk@1, CAST(cd1.cd_demo_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_hdemo_sk@2, ss_addr_sk@3, ss_promo_sk@4, ss_wholesale_cost@5, ss_list_price@6, ss_coupon_amt@7, d_year@8, s_store_name@9, s_zip@10, c_current_cdemo_sk@11, c_current_hdemo_sk@12, c_current_addr_sk@13, d_year@14, d_year@15, cd_marital_status@17] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_first_shipto_date_sk@14, CAST(d3.d_date_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_cdemo_sk@1, ss_hdemo_sk@2, ss_addr_sk@3, ss_promo_sk@4, ss_wholesale_cost@5, ss_list_price@6, ss_coupon_amt@7, d_year@8, s_store_name@9, s_zip@10, c_current_cdemo_sk@11, c_current_hdemo_sk@12, c_current_addr_sk@13, d_year@15, d_year@17] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_first_sales_date_sk@15, CAST(d2.d_date_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_cdemo_sk@1, ss_hdemo_sk@2, ss_addr_sk@3, ss_promo_sk@4, ss_wholesale_cost@5, ss_list_price@6, ss_coupon_amt@7, d_year@8, s_store_name@9, s_zip@10, c_current_cdemo_sk@11, c_current_hdemo_sk@12, c_current_addr_sk@13, c_first_shipto_date_sk@14, d_year@17] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@1, CAST(customer.c_customer_sk AS Float64)@6)], projection=[ss_item_sk@0, ss_cdemo_sk@2, ss_hdemo_sk@3, ss_addr_sk@4, ss_promo_sk@5, ss_wholesale_cost@6, ss_list_price@7, ss_coupon_amt@8, d_year@9, s_store_name@10, s_zip@11, c_current_cdemo_sk@13, c_current_hdemo_sk@14, c_current_addr_sk@15, c_first_shipto_date_sk@16, c_first_sales_date_sk@17] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_item_sk@2 as ss_item_sk, ss_customer_sk@3 as ss_customer_sk, ss_cdemo_sk@4 as ss_cdemo_sk, ss_hdemo_sk@5 as ss_hdemo_sk, ss_addr_sk@6 as ss_addr_sk, ss_promo_sk@7 as ss_promo_sk, ss_wholesale_cost@8 as ss_wholesale_cost, ss_list_price@9 as ss_list_price, ss_coupon_amt@10 as ss_coupon_amt, d_year@11 as d_year, s_store_name@0 as s_store_name, s_zip@1 as s_zip] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@3, ss_store_sk@5)], projection=[s_store_name@1, s_zip@2, ss_item_sk@4, ss_customer_sk@5, ss_cdemo_sk@6, ss_hdemo_sk@7, ss_addr_sk@8, ss_promo_sk@10, ss_wholesale_cost@11, ss_list_price@12, ss_coupon_amt@13, d_year@14] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_customer_sk@2 as ss_customer_sk, ss_cdemo_sk@3 as ss_cdemo_sk, ss_hdemo_sk@4 as ss_hdemo_sk, ss_addr_sk@5 as ss_addr_sk, ss_store_sk@6 as ss_store_sk, ss_promo_sk@7 as ss_promo_sk, ss_wholesale_cost@8 as ss_wholesale_cost, ss_list_price@9 as ss_list_price, ss_coupon_amt@10 as ss_coupon_amt, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_customer_sk@5, ss_cdemo_sk@6, ss_hdemo_sk@7, ss_addr_sk@8, ss_store_sk@9, ss_promo_sk@10, ss_wholesale_cost@11, ss_list_price@12, ss_coupon_amt@13] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, ss_item_sk@1)], projection=[ss_sold_date_sk@1, ss_item_sk@2, ss_customer_sk@3, ss_cdemo_sk@4, ss_hdemo_sk@5, ss_addr_sk@6, ss_store_sk@7, ss_promo_sk@8, ss_wholesale_cost@9, ss_list_price@10, ss_coupon_amt@11] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CAST(sum(catalog_sales.cs_ext_list_price)@1 AS Decimal128(38, 2)) > Some(2),20,0 * sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)@2, projection=[cs_item_sk@0] + │ AggregateExec: mode=FinalPartitioned, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(catalog_sales.cs_ext_list_price), sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)] + │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_item_sk@0, ss_item_sk@1), (sr_ticket_number@1, ss_ticket_number@8)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_customer_sk@4, ss_cdemo_sk@5, ss_hdemo_sk@6, ss_addr_sk@7, ss_store_sk@8, ss_promo_sk@9, ss_wholesale_cost@11, ss_list_price@12, ss_coupon_amt@13] + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_cdemo_sk@1 as c_current_cdemo_sk, c_current_hdemo_sk@2 as c_current_hdemo_sk, c_current_addr_sk@3 as c_current_addr_sk, c_first_shipto_date_sk@4 as c_first_shipto_date_sk, c_first_sales_date_sk@5 as c_first_sales_date_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, cd_marital_status@1 as cd_marital_status, CAST(cd_demo_sk@0 AS Float64) as CAST(cd1.cd_demo_sk AS Float64)] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, cd_marital_status@1 as cd_marital_status, CAST(cd_demo_sk@0 AS Float64) as CAST(cd2.cd_demo_sk AS Float64)] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, hd_income_band_sk@1 as hd_income_band_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(hd1.hd_demo_sk AS Float64)] + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, hd_income_band_sk@1 as hd_income_band_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(hd2.hd_demo_sk AS Float64)] + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_street_number@1 as ca_street_number, ca_street_name@2 as ca_street_name, ca_city@3 as ca_city, ca_zip@4 as ca_zip, CAST(ca_address_sk@0 AS Float64) as CAST(ad1.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet]]}, projection=[ib_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_color@2 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@1 >= Some(6500),4,2 AND i_current_price@1 <= Some(7400),4,2, projection=[i_item_sk@0, i_product_name@3] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price, i_color, i_product_name], file_type=parquet, predicate=i_color@2 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@1 >= Some(6500),4,2 AND i_current_price@1 <= Some(7400),4,2 AND DynamicFilter [ empty ], pruning_predicate=(i_color_null_count@2 != row_count@3 AND i_color_min@0 <= purple AND purple <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burlywood AND burlywood <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= indian AND indian <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= spring AND spring <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= floral AND floral <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= medium AND medium <= i_color_max@1) AND i_current_price_null_count@5 != row_count@3 AND i_current_price_max@4 >= Some(6500),4,2 AND i_current_price_null_count@5 != row_count@3 AND i_current_price_min@6 <= Some(7400),4,2, required_guarantees=[i_color in (burlywood, floral, indian, medium, purple, spring)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_name@1 as s_store_name, s_zip@2 as s_zip, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_zip], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1999 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(catalog_sales.cs_ext_list_price), sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)] + │ ProjectionExec: expr=[cs_item_sk@3 as cs_item_sk, cs_ext_list_price@4 as cs_ext_list_price, cr_refunded_cash@0 as cr_refunded_cash, cr_reversed_charge@1 as cr_reversed_charge, cr_store_credit@2 as cr_store_credit] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cr_item_sk@0, cs_item_sk@0), (cr_order_number@1, cs_order_number@1)], projection=[cr_refunded_cash@2, cr_reversed_charge@3, cr_store_credit@4, cs_item_sk@5, cs_ext_list_price@7] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cr_item_sk@0, cr_order_number@1], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_refunded_cash, cr_reversed_charge, cr_store_credit], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@0, cs_order_number@1], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@1, ss_ticket_number@8], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_name@1 as s_store_name, s_zip@2 as s_zip, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_zip], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2000 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(catalog_sales.cs_ext_list_price), sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)] + │ ProjectionExec: expr=[cs_item_sk@3 as cs_item_sk, cs_ext_list_price@4 as cs_ext_list_price, cr_refunded_cash@0 as cr_refunded_cash, cr_reversed_charge@1 as cr_reversed_charge, cr_store_credit@2 as cr_store_credit] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cr_item_sk@0, cs_item_sk@0), (cr_order_number@1, cs_order_number@1)], projection=[cr_refunded_cash@2, cr_reversed_charge@3, cr_store_credit@4, cs_item_sk@5, cs_ext_list_price@7] + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cr_item_sk@0, cr_order_number@1], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_refunded_cash, cr_reversed_charge, cr_store_credit], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@0, cs_order_number@1], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@1, ss_ticket_number@8], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_65() -> Result<()> { + let display = test_tpcds_query(65).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [s_store_name@0 ASC, i_item_desc@1 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[s_store_name@0 ASC, i_item_desc@1 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[s_store_name@0 as s_store_name, i_item_desc@2 as i_item_desc, revenue@1 as revenue, i_current_price@3 as i_current_price, i_wholesale_cost@4 as i_wholesale_cost, i_brand@5 as i_brand] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_store_sk@1, ss_store_sk@0)], filter=CAST(revenue@0 AS Decimal128(30, 15)) <= CAST(0.1 * CAST(ave@1 AS Float64) AS Decimal128(30, 15)), projection=[s_store_name@0, revenue@2, i_item_desc@3, i_current_price@4, i_wholesale_cost@5, i_brand@6] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@2, i_item_sk@0)], projection=[s_store_name@0, ss_store_sk@1, revenue@3, i_item_desc@5, i_current_price@6, i_wholesale_cost@7, i_brand@8] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, ss_store_sk@0)], projection=[s_store_name@1, ss_store_sk@3, ss_item_sk@4, revenue@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[ss_store_sk@0 as ss_store_sk, ss_item_sk@1 as ss_item_sk, sum(store_sales.ss_sales_price)@2 as revenue] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_store_sk@0 as ss_store_sk, ss_item_sk@1 as ss_item_sk], aggr=[sum(store_sales.ss_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_store_sk@0, ss_item_sk@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_store_sk@1 as ss_store_sk, ss_item_sk@0 as ss_item_sk], aggr=[sum(store_sales.ss_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_desc, i_current_price, i_wholesale_cost, i_brand], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ss_store_sk@0 as ss_store_sk, avg(sa.revenue)@1 as ave] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(sa.revenue)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_store_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(sa.revenue)] + │ ProjectionExec: expr=[ss_store_sk@0 as ss_store_sk, sum(store_sales.ss_sales_price)@2 as revenue] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_store_sk@0 as ss_store_sk, ss_item_sk@1 as ss_item_sk], aggr=[sum(store_sales.ss_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_store_sk@0, ss_item_sk@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_store_sk@1 as ss_store_sk, ss_item_sk@0 as ss_item_sk], aggr=[sum(store_sales.ss_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_name@1 as s_store_name, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_month_seq@1 >= 1176 AND d_month_seq@1 <= 1187, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@1 >= 1176 AND d_month_seq@1 <= 1187, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1176 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1187, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_month_seq@1 >= 1176 AND d_month_seq@1 <= 1187, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@1 >= 1176 AND d_month_seq@1 <= 1187, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1176 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1187, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_66() -> Result<()> { + let display = test_tpcds_query(66).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [w_warehouse_name@0 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[w_warehouse_name@0 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, ship_carriers@6 as ship_carriers, year_@7 as year_, sum(x.jan_sales)@8 as jan_sales, sum(x.feb_sales)@9 as feb_sales, sum(x.mar_sales)@10 as mar_sales, sum(x.apr_sales)@11 as apr_sales, sum(x.may_sales)@12 as may_sales, sum(x.jun_sales)@13 as jun_sales, sum(x.jul_sales)@14 as jul_sales, sum(x.aug_sales)@15 as aug_sales, sum(x.sep_sales)@16 as sep_sales, sum(x.oct_sales)@17 as oct_sales, sum(x.nov_sales)@18 as nov_sales, sum(x.dec_sales)@19 as dec_sales, sum(x.jan_sales / x.w_warehouse_sq_ft)@20 as jan_sales_per_sq_foot, sum(x.feb_sales / x.w_warehouse_sq_ft)@21 as feb_sales_per_sq_foot, sum(x.mar_sales / x.w_warehouse_sq_ft)@22 as mar_sales_per_sq_foot, sum(x.apr_sales / x.w_warehouse_sq_ft)@23 as apr_sales_per_sq_foot, sum(x.may_sales / x.w_warehouse_sq_ft)@24 as may_sales_per_sq_foot, sum(x.jun_sales / x.w_warehouse_sq_ft)@25 as jun_sales_per_sq_foot, sum(x.jul_sales / x.w_warehouse_sq_ft)@26 as jul_sales_per_sq_foot, sum(x.aug_sales / x.w_warehouse_sq_ft)@27 as aug_sales_per_sq_foot, sum(x.sep_sales / x.w_warehouse_sq_ft)@28 as sep_sales_per_sq_foot, sum(x.oct_sales / x.w_warehouse_sq_ft)@29 as oct_sales_per_sq_foot, sum(x.nov_sales / x.w_warehouse_sq_ft)@30 as nov_sales_per_sq_foot, sum(x.dec_sales / x.w_warehouse_sq_ft)@31 as dec_sales_per_sq_foot, sum(x.jan_net)@32 as jan_net, sum(x.feb_net)@33 as feb_net, sum(x.mar_net)@34 as mar_net, sum(x.apr_net)@35 as apr_net, sum(x.may_net)@36 as may_net, sum(x.jun_net)@37 as jun_net, sum(x.jul_net)@38 as jul_net, sum(x.aug_net)@39 as aug_net, sum(x.sep_net)@40 as sep_net, sum(x.oct_net)@41 as oct_net, sum(x.nov_net)@42 as nov_net, sum(x.dec_net)@43 as dec_net] + │ AggregateExec: mode=FinalPartitioned, gby=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, ship_carriers@6 as ship_carriers, year_@7 as year_], aggr=[sum(x.jan_sales), sum(x.feb_sales), sum(x.mar_sales), sum(x.apr_sales), sum(x.may_sales), sum(x.jun_sales), sum(x.jul_sales), sum(x.aug_sales), sum(x.sep_sales), sum(x.oct_sales), sum(x.nov_sales), sum(x.dec_sales), sum(x.jan_sales / x.w_warehouse_sq_ft), sum(x.feb_sales / x.w_warehouse_sq_ft), sum(x.mar_sales / x.w_warehouse_sq_ft), sum(x.apr_sales / x.w_warehouse_sq_ft), sum(x.may_sales / x.w_warehouse_sq_ft), sum(x.jun_sales / x.w_warehouse_sq_ft), sum(x.jul_sales / x.w_warehouse_sq_ft), sum(x.aug_sales / x.w_warehouse_sq_ft), sum(x.sep_sales / x.w_warehouse_sq_ft), sum(x.oct_sales / x.w_warehouse_sq_ft), sum(x.nov_sales / x.w_warehouse_sq_ft), sum(x.dec_sales / x.w_warehouse_sq_ft), sum(x.jan_net), sum(x.feb_net), sum(x.mar_net), sum(x.apr_net), sum(x.may_net), sum(x.jun_net), sum(x.jul_net), sum(x.aug_net), sum(x.sep_net), sum(x.oct_net), sum(x.nov_net), sum(x.dec_net)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([w_warehouse_name@0, w_warehouse_sq_ft@1, w_city@2, w_county@3, w_state@4, w_country@5, ship_carriers@6, year_@7], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[w_warehouse_name@1 as w_warehouse_name, w_warehouse_sq_ft@2 as w_warehouse_sq_ft, w_city@3 as w_city, w_county@4 as w_county, w_state@5 as w_state, w_country@6 as w_country, ship_carriers@7 as ship_carriers, year_@8 as year_], aggr=[sum(x.jan_sales), sum(x.feb_sales), sum(x.mar_sales), sum(x.apr_sales), sum(x.may_sales), sum(x.jun_sales), sum(x.jul_sales), sum(x.aug_sales), sum(x.sep_sales), sum(x.oct_sales), sum(x.nov_sales), sum(x.dec_sales), sum(x.jan_sales / x.w_warehouse_sq_ft), sum(x.feb_sales / x.w_warehouse_sq_ft), sum(x.mar_sales / x.w_warehouse_sq_ft), sum(x.apr_sales / x.w_warehouse_sq_ft), sum(x.may_sales / x.w_warehouse_sq_ft), sum(x.jun_sales / x.w_warehouse_sq_ft), sum(x.jul_sales / x.w_warehouse_sq_ft), sum(x.aug_sales / x.w_warehouse_sq_ft), sum(x.sep_sales / x.w_warehouse_sq_ft), sum(x.oct_sales / x.w_warehouse_sq_ft), sum(x.nov_sales / x.w_warehouse_sq_ft), sum(x.dec_sales / x.w_warehouse_sq_ft), sum(x.jan_net), sum(x.feb_net), sum(x.mar_net), sum(x.apr_net), sum(x.may_net), sum(x.jun_net), sum(x.jul_net), sum(x.aug_net), sum(x.sep_net), sum(x.oct_net), sum(x.nov_net), sum(x.dec_net)] + │ ProjectionExec: expr=[CAST(w_warehouse_sq_ft@1 AS Float64) as __common_expr_1, w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, ship_carriers@6 as ship_carriers, year_@7 as year_, jan_sales@8 as jan_sales, feb_sales@9 as feb_sales, mar_sales@10 as mar_sales, apr_sales@11 as apr_sales, may_sales@12 as may_sales, jun_sales@13 as jun_sales, jul_sales@14 as jul_sales, aug_sales@15 as aug_sales, sep_sales@16 as sep_sales, oct_sales@17 as oct_sales, nov_sales@18 as nov_sales, dec_sales@19 as dec_sales, jan_net@20 as jan_net, feb_net@21 as feb_net, mar_net@22 as mar_net, apr_net@23 as apr_net, may_net@24 as may_net, jun_net@25 as jun_net, jul_net@26 as jul_net, aug_net@27 as aug_net, sep_net@28 as sep_net, oct_net@29 as oct_net, nov_net@30 as nov_net, dec_net@31 as dec_net] + │ InterleaveExec + │ ProjectionExec: expr=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, DHL,BARIAN as ship_carriers, d_year@6 as year_, sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@7 as jan_sales, sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@8 as feb_sales, sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@9 as mar_sales, sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@10 as apr_sales, sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@11 as may_sales, sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@12 as jun_sales, sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@13 as jul_sales, sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@14 as aug_sales, sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@15 as sep_sales, sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@16 as oct_sales, sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@17 as nov_sales, sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@18 as dec_sales, sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@19 as jan_net, sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@20 as feb_net, sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@21 as mar_net, sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@22 as apr_net, sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@23 as may_net, sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@24 as jun_net, sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@25 as jul_net, sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@26 as aug_net, sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@27 as sep_net, sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@28 as oct_net, sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@29 as nov_net, sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@30 as dec_net] + │ AggregateExec: mode=FinalPartitioned, gby=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, d_year@6 as d_year], aggr=[sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)], ordering_mode=PartiallySorted([6]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([w_warehouse_name@0, w_warehouse_sq_ft@1, w_city@2, w_county@3, w_state@4, w_country@5, d_year@6], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[w_warehouse_name@15 as w_warehouse_name, w_warehouse_sq_ft@16 as w_warehouse_sq_ft, w_city@17 as w_city, w_county@18 as w_county, w_state@19 as w_state, w_country@20 as w_country, d_year@21 as d_year], aggr=[sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)], ordering_mode=PartiallySorted([6]) + │ ProjectionExec: expr=[d_moy@10 = 1 as __common_expr_2, d_moy@10 = 2 as __common_expr_3, d_moy@10 = 3 as __common_expr_4, d_moy@10 = 4 as __common_expr_5, d_moy@10 = 5 as __common_expr_6, d_moy@10 = 6 as __common_expr_7, d_moy@10 = 7 as __common_expr_8, d_moy@10 = 8 as __common_expr_9, d_moy@10 = 9 as __common_expr_10, d_moy@10 = 10 as __common_expr_11, d_moy@10 = 11 as __common_expr_12, d_moy@10 = 12 as __common_expr_13, ws_quantity@0 as ws_quantity, ws_ext_sales_price@1 as ws_ext_sales_price, ws_net_paid@2 as ws_net_paid, w_warehouse_name@3 as w_warehouse_name, w_warehouse_sq_ft@4 as w_warehouse_sq_ft, w_city@5 as w_city, w_county@6 as w_county, w_state@7 as w_state, w_country@8 as w_country, d_year@9 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(ship_mode.sm_ship_mode_sk AS Float64)@1, ws_ship_mode_sk@0)], projection=[ws_quantity@3, ws_ext_sales_price@4, ws_net_paid@5, w_warehouse_name@6, w_warehouse_sq_ft@7, w_city@8, w_county@9, w_state@10, w_country@11, d_year@12, d_moy@13] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_sold_time_sk@0, CAST(time_dim.t_time_sk AS Float64)@1)], projection=[ws_ship_mode_sk@1, ws_quantity@2, ws_ext_sales_price@3, ws_net_paid@4, w_warehouse_name@5, w_warehouse_sq_ft@6, w_city@7, w_county@8, w_state@9, w_country@10, d_year@11, d_moy@12] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_sold_date_sk@0, CAST(date_dim.d_date_sk AS Float64)@3)], projection=[ws_sold_time_sk@1, ws_ship_mode_sk@2, ws_quantity@3, ws_ext_sales_price@4, ws_net_paid@5, w_warehouse_name@6, w_warehouse_sq_ft@7, w_city@8, w_county@9, w_state@10, w_country@11, d_year@13, d_moy@14] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ws_sold_date_sk@6 as ws_sold_date_sk, ws_sold_time_sk@7 as ws_sold_time_sk, ws_ship_mode_sk@8 as ws_ship_mode_sk, ws_quantity@9 as ws_quantity, ws_ext_sales_price@10 as ws_ext_sales_price, ws_net_paid@11 as ws_net_paid, w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(warehouse.w_warehouse_sk AS Float64)@7, ws_warehouse_sk@3)], projection=[w_warehouse_name@1, w_warehouse_sq_ft@2, w_city@3, w_county@4, w_state@5, w_country@6, ws_sold_date_sk@8, ws_sold_time_sk@9, ws_ship_mode_sk@10, ws_quantity@12, ws_ext_sales_price@13, ws_net_paid@14] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_sold_time_sk, ws_ship_mode_sk, ws_warehouse_sk, ws_quantity, ws_ext_sales_price, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_moy@2 as d_moy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: t_time@1 >= 30838 AND t_time@1 <= 59638, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_time], file_type=parquet, predicate=t_time@1 >= 30838 AND t_time@1 <= 59638, pruning_predicate=t_time_null_count@1 != row_count@2 AND t_time_max@0 >= 30838 AND t_time_null_count@1 != row_count@2 AND t_time_min@3 <= 59638, required_guarantees=[] + │ ProjectionExec: expr=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, DHL,BARIAN as ship_carriers, d_year@6 as year_, sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@7 as jan_sales, sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@8 as feb_sales, sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@9 as mar_sales, sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@10 as apr_sales, sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@11 as may_sales, sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@12 as jun_sales, sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@13 as jul_sales, sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@14 as aug_sales, sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@15 as sep_sales, sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@16 as oct_sales, sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@17 as nov_sales, sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@18 as dec_sales, sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@19 as jan_net, sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@20 as feb_net, sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@21 as mar_net, sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@22 as apr_net, sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@23 as may_net, sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@24 as jun_net, sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@25 as jul_net, sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@26 as aug_net, sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@27 as sep_net, sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@28 as oct_net, sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@29 as nov_net, sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@30 as dec_net] + │ AggregateExec: mode=FinalPartitioned, gby=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, d_year@6 as d_year], aggr=[sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)], ordering_mode=PartiallySorted([6]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([w_warehouse_name@0, w_warehouse_sq_ft@1, w_city@2, w_county@3, w_state@4, w_country@5, d_year@6], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[w_warehouse_name@15 as w_warehouse_name, w_warehouse_sq_ft@16 as w_warehouse_sq_ft, w_city@17 as w_city, w_county@18 as w_county, w_state@19 as w_state, w_country@20 as w_country, d_year@21 as d_year], aggr=[sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)], ordering_mode=PartiallySorted([6]) + │ ProjectionExec: expr=[d_moy@10 = 1 as __common_expr_14, d_moy@10 = 2 as __common_expr_15, d_moy@10 = 3 as __common_expr_16, d_moy@10 = 4 as __common_expr_17, d_moy@10 = 5 as __common_expr_18, d_moy@10 = 6 as __common_expr_19, d_moy@10 = 7 as __common_expr_20, d_moy@10 = 8 as __common_expr_21, d_moy@10 = 9 as __common_expr_22, d_moy@10 = 10 as __common_expr_23, d_moy@10 = 11 as __common_expr_24, d_moy@10 = 12 as __common_expr_25, cs_quantity@0 as cs_quantity, cs_sales_price@1 as cs_sales_price, cs_net_paid_inc_tax@2 as cs_net_paid_inc_tax, w_warehouse_name@3 as w_warehouse_name, w_warehouse_sq_ft@4 as w_warehouse_sq_ft, w_city@5 as w_city, w_county@6 as w_county, w_state@7 as w_state, w_country@8 as w_country, d_year@9 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(ship_mode.sm_ship_mode_sk AS Float64)@1, cs_ship_mode_sk@0)], projection=[cs_quantity@3, cs_sales_price@4, cs_net_paid_inc_tax@5, w_warehouse_name@6, w_warehouse_sq_ft@7, w_city@8, w_county@9, w_state@10, w_country@11, d_year@12, d_moy@13] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_sold_time_sk@0, CAST(time_dim.t_time_sk AS Float64)@1)], projection=[cs_ship_mode_sk@1, cs_quantity@2, cs_sales_price@3, cs_net_paid_inc_tax@4, w_warehouse_name@5, w_warehouse_sq_ft@6, w_city@7, w_county@8, w_state@9, w_country@10, d_year@11, d_moy@12] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_sold_date_sk@0, CAST(date_dim.d_date_sk AS Float64)@3)], projection=[cs_sold_time_sk@1, cs_ship_mode_sk@2, cs_quantity@3, cs_sales_price@4, cs_net_paid_inc_tax@5, w_warehouse_name@6, w_warehouse_sq_ft@7, w_city@8, w_county@9, w_state@10, w_country@11, d_year@13, d_moy@14] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[cs_sold_date_sk@6 as cs_sold_date_sk, cs_sold_time_sk@7 as cs_sold_time_sk, cs_ship_mode_sk@8 as cs_ship_mode_sk, cs_quantity@9 as cs_quantity, cs_sales_price@10 as cs_sales_price, cs_net_paid_inc_tax@11 as cs_net_paid_inc_tax, w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(warehouse.w_warehouse_sk AS Float64)@7, cs_warehouse_sk@3)], projection=[w_warehouse_name@1, w_warehouse_sq_ft@2, w_city@3, w_county@4, w_state@5, w_country@6, cs_sold_date_sk@8, cs_sold_time_sk@9, cs_ship_mode_sk@10, cs_quantity@12, cs_sales_price@13, cs_net_paid_inc_tax@14] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_sold_time_sk, cs_ship_mode_sk, cs_warehouse_sk, cs_quantity, cs_sales_price, cs_net_paid_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_moy@2 as d_moy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: t_time@1 >= 30838 AND t_time@1 <= 59638, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_time], file_type=parquet, predicate=t_time@1 >= 30838 AND t_time@1 <= 59638, pruning_predicate=t_time_null_count@1 != row_count@2 AND t_time_max@0 >= 30838 AND t_time_null_count@1 != row_count@2 AND t_time_min@3 <= 59638, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[sm_ship_mode_sk@0 as sm_ship_mode_sk, CAST(sm_ship_mode_sk@0 AS Float64) as CAST(ship_mode.sm_ship_mode_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: sm_carrier@1 = DHL OR sm_carrier@1 = BARIAN, projection=[sm_ship_mode_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-3.parquet]]}, projection=[sm_ship_mode_sk, sm_carrier], file_type=parquet, predicate=sm_carrier@1 = DHL OR sm_carrier@1 = BARIAN, pruning_predicate=sm_carrier_null_count@2 != row_count@3 AND sm_carrier_min@0 <= DHL AND DHL <= sm_carrier_max@1 OR sm_carrier_null_count@2 != row_count@3 AND sm_carrier_min@0 <= BARIAN AND BARIAN <= sm_carrier_max@1, required_guarantees=[sm_carrier in (BARIAN, DHL)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[w_warehouse_sk@0 as w_warehouse_sk, w_warehouse_name@1 as w_warehouse_name, w_warehouse_sq_ft@2 as w_warehouse_sq_ft, w_city@3 as w_city, w_county@4 as w_county, w_state@5 as w_state, w_country@6 as w_country, CAST(w_warehouse_sk@0 AS Float64) as CAST(warehouse.w_warehouse_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet]]}, projection=[w_warehouse_sk, w_warehouse_name, w_warehouse_sq_ft, w_city, w_county, w_state, w_country], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[sm_ship_mode_sk@0 as sm_ship_mode_sk, CAST(sm_ship_mode_sk@0 AS Float64) as CAST(ship_mode.sm_ship_mode_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: sm_carrier@1 = DHL OR sm_carrier@1 = BARIAN, projection=[sm_ship_mode_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-3.parquet]]}, projection=[sm_ship_mode_sk, sm_carrier], file_type=parquet, predicate=sm_carrier@1 = DHL OR sm_carrier@1 = BARIAN, pruning_predicate=sm_carrier_null_count@2 != row_count@3 AND sm_carrier_min@0 <= DHL AND DHL <= sm_carrier_max@1 OR sm_carrier_null_count@2 != row_count@3 AND sm_carrier_min@0 <= BARIAN AND BARIAN <= sm_carrier_max@1, required_guarantees=[sm_carrier in (BARIAN, DHL)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[w_warehouse_sk@0 as w_warehouse_sk, w_warehouse_name@1 as w_warehouse_name, w_warehouse_sq_ft@2 as w_warehouse_sq_ft, w_city@3 as w_city, w_county@4 as w_county, w_state@5 as w_state, w_country@6 as w_country, CAST(w_warehouse_sk@0 AS Float64) as CAST(warehouse.w_warehouse_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet]]}, projection=[w_warehouse_sk, w_warehouse_name, w_warehouse_sq_ft, w_city, w_county, w_state, w_country], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_67() -> Result<()> { + let display = test_tpcds_query(67).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_category@0 ASC, i_class@1 ASC, i_brand@2 ASC, i_product_name@3 ASC, d_year@4 ASC, d_qoy@5 ASC, d_moy@6 ASC, s_store_id@7 ASC, sumsales@8 ASC, rk@9 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_category@0 ASC, i_class@1 ASC, i_brand@2 ASC, i_product_name@3 ASC, d_year@4 ASC, d_qoy@5 ASC, d_moy@6 ASC, s_store_id@7 ASC, sumsales@8 ASC, rk@9 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_category@0 as i_category, i_class@1 as i_class, i_brand@2 as i_brand, i_product_name@3 as i_product_name, d_year@4 as d_year, d_qoy@5 as d_qoy, d_moy@6 as d_moy, s_store_id@7 as s_store_id, sumsales@8 as sumsales, rank() PARTITION BY [dw1.i_category] ORDER BY [dw1.sumsales DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@9 as rk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: rank() PARTITION BY [dw1.i_category] ORDER BY [dw1.sumsales DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@9 <= 100 + │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [dw1.i_category] ORDER BY [dw1.sumsales DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [dw1.i_category] ORDER BY [dw1.sumsales DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, sumsales@8 DESC], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0], 3), input_partitions=3 + │ ProjectionExec: expr=[i_category@0 as i_category, i_class@1 as i_class, i_brand@2 as i_brand, i_product_name@3 as i_product_name, d_year@4 as d_year, d_qoy@5 as d_qoy, d_moy@6 as d_moy, s_store_id@7 as s_store_id, sum(coalesce(store_sales.ss_sales_price * store_sales.ss_quantity,Int64(0)))@9 as sumsales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_class@1 as i_class, i_brand@2 as i_brand, i_product_name@3 as i_product_name, d_year@4 as d_year, d_qoy@5 as d_qoy, d_moy@6 as d_moy, s_store_id@7 as s_store_id, __grouping_id@8 as __grouping_id], aggr=[sum(coalesce(store_sales.ss_sales_price * store_sales.ss_quantity,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_class@1, i_brand@2, i_product_name@3, d_year@4, d_qoy@5, d_moy@6, s_store_id@7, __grouping_id@8], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[(NULL as i_category, NULL as i_class, NULL as i_brand, NULL as i_product_name, NULL as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@8 as i_category, NULL as i_class, NULL as i_brand, NULL as i_product_name, NULL as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@8 as i_category, i_class@7 as i_class, NULL as i_brand, NULL as i_product_name, NULL as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@8 as i_category, i_class@7 as i_class, i_brand@6 as i_brand, NULL as i_product_name, NULL as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@8 as i_category, i_class@7 as i_class, i_brand@6 as i_brand, i_product_name@9 as i_product_name, NULL as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@8 as i_category, i_class@7 as i_class, i_brand@6 as i_brand, i_product_name@9 as i_product_name, d_year@2 as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@8 as i_category, i_class@7 as i_class, i_brand@6 as i_brand, i_product_name@9 as i_product_name, d_year@2 as d_year, d_qoy@4 as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@8 as i_category, i_class@7 as i_class, i_brand@6 as i_brand, i_product_name@9 as i_product_name, d_year@2 as d_year, d_qoy@4 as d_qoy, d_moy@3 as d_moy, NULL as s_store_id), (i_category@8 as i_category, i_class@7 as i_class, i_brand@6 as i_brand, i_product_name@9 as i_product_name, d_year@2 as d_year, d_qoy@4 as d_qoy, d_moy@3 as d_moy, s_store_id@5 as s_store_id)], aggr=[sum(coalesce(store_sales.ss_sales_price * store_sales.ss_quantity,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_quantity@1, ss_sales_price@2, d_year@3, d_moy@4, d_qoy@5, s_store_id@6, i_brand@8, i_class@9, i_category@10, i_product_name@11] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_quantity@2 as ss_quantity, ss_sales_price@3 as ss_sales_price, d_year@4 as d_year, d_moy@5 as d_moy, d_qoy@6 as d_qoy, s_store_id@0 as s_store_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, ss_store_sk@1)], projection=[s_store_id@1, ss_item_sk@3, ss_quantity@5, ss_sales_price@6, d_year@7, d_moy@8, d_qoy@9] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[ss_item_sk@3 as ss_item_sk, ss_store_sk@4 as ss_store_sk, ss_quantity@5 as ss_quantity, ss_sales_price@6 as ss_sales_price, d_year@0 as d_year, d_moy@1 as d_moy, d_qoy@2 as d_qoy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@4, ss_sold_date_sk@0)], projection=[d_year@1, d_moy@2, d_qoy@3, ss_item_sk@6, ss_store_sk@7, ss_quantity@8, ss_sales_price@9] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_product_name], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_id@1 as s_store_id, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_moy@2 as d_moy, d_qoy@3 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, projection=[d_date_sk@0, d_year@2, d_moy@3, d_qoy@4] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq, d_year, d_moy, d_qoy], file_type=parquet, predicate=d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_68() -> Result<()> { + let display = test_tpcds_query(68).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c_last_name@0 ASC, ss_ticket_number@4 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[c_last_name@0 ASC, ss_ticket_number@4 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[c_last_name@6 as c_last_name, c_first_name@5 as c_first_name, ca_city@7 as ca_city, bought_city@1 as bought_city, ss_ticket_number@0 as ss_ticket_number, extended_price@2 as extended_price, extended_tax@4 as extended_tax, list_price@3 as list_price] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@5, ca_address_sk@0)], filter=bought_city@0 != ca_city@1, projection=[ss_ticket_number@0, bought_city@1, extended_price@2, list_price@3, extended_tax@4, c_first_name@6, c_last_name@7, ca_city@9] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@1, CAST(customer.c_customer_sk AS Float64)@4)], projection=[ss_ticket_number@0, bought_city@2, extended_price@3, list_price@4, extended_tax@5, c_current_addr_sk@7, c_first_name@8, c_last_name@9] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, ca_city@3 as bought_city, sum(store_sales.ss_ext_sales_price)@4 as extended_price, sum(store_sales.ss_ext_list_price)@5 as list_price, sum(store_sales.ss_ext_tax)@6 as extended_tax] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, ss_addr_sk@2 as ss_addr_sk, ca_city@3 as ca_city], aggr=[sum(store_sales.ss_ext_sales_price), sum(store_sales.ss_ext_list_price), sum(store_sales.ss_ext_tax)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_ticket_number@0, ss_customer_sk@1, ss_addr_sk@2, ca_city@3], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_ticket_number@2 as ss_ticket_number, ss_customer_sk@0 as ss_customer_sk, ss_addr_sk@1 as ss_addr_sk, ca_city@6 as ca_city], aggr=[sum(store_sales.ss_ext_sales_price), sum(store_sales.ss_ext_list_price), sum(store_sales.ss_ext_tax)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@2)], projection=[ss_customer_sk@0, ss_addr_sk@1, ss_ticket_number@2, ss_ext_sales_price@3, ss_ext_list_price@4, ss_ext_tax@5, ca_city@7] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_customer_sk@2, ss_addr_sk@4, ss_ticket_number@5, ss_ext_sales_price@6, ss_ext_list_price@7, ss_ext_tax@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@3)], projection=[ss_customer_sk@2, ss_hdemo_sk@3, ss_addr_sk@4, ss_ticket_number@6, ss_ext_sales_price@7, ss_ext_list_price@8, ss_ext_tax@9] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_customer_sk@3, ss_hdemo_sk@4, ss_addr_sk@5, ss_store_sk@6, ss_ticket_number@7, ss_ext_sales_price@8, ss_ext_list_price@9, ss_ext_tax@10] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_ext_sales_price, ss_ext_list_price, ss_ext_tax], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_city@1 as ca_city, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_city], file_type=parquet + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_city], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: hd_dep_count@1 = 4 OR hd_vehicle_count@2 = 3, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@1 = 4 OR hd_vehicle_count@2 = 3, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 4 AND 4 <= hd_dep_count_max@1 OR hd_vehicle_count_null_count@6 != row_count@3 AND hd_vehicle_count_min@4 <= 3 AND 3 <= hd_vehicle_count_max@5, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_city@1 = Fairview OR s_city@1 = Midway, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_city], file_type=parquet, predicate=s_city@1 = Fairview OR s_city@1 = Midway, pruning_predicate=s_city_null_count@2 != row_count@3 AND s_city_min@0 <= Fairview AND Fairview <= s_city_max@1 OR s_city_null_count@2 != row_count@3 AND s_city_min@0 <= Midway AND Midway <= s_city_max@1, required_guarantees=[s_city in (Fairview, Midway)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_dom@2 >= 1 AND d_dom@2 <= 2 AND (d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001), projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=d_dom@2 >= 1 AND d_dom@2 <= 2 AND (d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001), pruning_predicate=d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 2 AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_69() -> Result<()> { + let display = test_tpcds_query(69).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [cd_gender@0 ASC NULLS LAST, cd_marital_status@1 ASC NULLS LAST, cd_education_status@2 ASC NULLS LAST, cd_purchase_estimate@4 ASC NULLS LAST, cd_credit_rating@6 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[cd_gender@0 ASC NULLS LAST, cd_marital_status@1 ASC NULLS LAST, cd_education_status@2 ASC NULLS LAST, cd_purchase_estimate@4 ASC NULLS LAST, cd_credit_rating@6 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[cd_gender@0 as cd_gender, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, count(Int64(1))@5 as cnt1, cd_purchase_estimate@3 as cd_purchase_estimate, count(Int64(1))@5 as cnt2, cd_credit_rating@4 as cd_credit_rating, count(Int64(1))@5 as cnt3] + │ AggregateExec: mode=FinalPartitioned, gby=[cd_gender@0 as cd_gender, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, cd_purchase_estimate@3 as cd_purchase_estimate, cd_credit_rating@4 as cd_credit_rating], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cd_gender@0, cd_marital_status@1, cd_education_status@2, cd_purchase_estimate@3, cd_credit_rating@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cd_gender@0 as cd_gender, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, cd_purchase_estimate@3 as cd_purchase_estimate, cd_credit_rating@4 as cd_credit_rating], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(cs_ship_customer_sk@0, CAST(c.c_customer_sk AS Float64)@6)], projection=[cd_gender@1, cd_marital_status@2, cd_education_status@3, cd_purchase_estimate@4, cd_credit_rating@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_ship_customer_sk@3] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_education_status@3 as cd_education_status, cd_purchase_estimate@4 as cd_purchase_estimate, cd_credit_rating@5 as cd_credit_rating, CAST(c_customer_sk@0 AS Float64) as CAST(c.c_customer_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(ws_bill_customer_sk@0, CAST(c.c_customer_sk AS Float64)@6)], projection=[c_customer_sk@0, cd_gender@1, cd_marital_status@2, cd_education_status@3, cd_purchase_estimate@4, cd_credit_rating@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_bill_customer_sk@3] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_education_status@3 as cd_education_status, cd_purchase_estimate@4 as cd_purchase_estimate, cd_credit_rating@5 as cd_credit_rating, CAST(c_customer_sk@0 AS Float64) as CAST(c.c_customer_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(ss_customer_sk@0, CAST(c.c_customer_sk AS Float64)@6)], projection=[c_customer_sk@0, cd_gender@1, cd_marital_status@2, cd_education_status@3, cd_purchase_estimate@4, cd_credit_rating@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_customer_sk@3] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_education_status@3 as cd_education_status, cd_purchase_estimate@4 as cd_purchase_estimate, cd_credit_rating@5 as cd_credit_rating, CAST(c_customer_sk@0 AS Float64) as CAST(c.c_customer_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@1, CAST(customer_demographics.cd_demo_sk AS Float64)@6)], projection=[c_customer_sk@0, cd_gender@3, cd_marital_status@4, cd_education_status@5, cd_purchase_estimate@6, cd_credit_rating@7] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@2)], projection=[c_customer_sk@1, c_current_cdemo_sk@2] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_education_status@3 as cd_education_status, cd_purchase_estimate@4 as cd_purchase_estimate, cd_credit_rating@5 as cd_credit_rating, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 AND d_moy@2 >= 4 AND d_moy@2 <= 6, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 >= 4 AND d_moy@2 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 AND d_moy@2 >= 4 AND d_moy@2 <= 6, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 >= 4 AND d_moy@2 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 AND d_moy@2 >= 4 AND d_moy@2 <= 6, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 >= 4 AND d_moy@2 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ca_state@1 = KY OR ca_state@1 = GA OR ca_state@1 = NM, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@1 = KY OR ca_state@1 = GA OR ca_state@1 = NM, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= KY AND KY <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NM AND NM <= ca_state_max@1, required_guarantees=[ca_state in (GA, KY, NM)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + #[ignore = "The ordering of the column names in the first nodes is non deterministickI"] + async fn test_tpcds_70() -> Result<()> { + let display = test_tpcds_query(70).await?; + assert_snapshot!(display, @r#""#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_71() -> Result<()> { + let display = test_tpcds_query(71).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[brand_id@0 as brand_id, brand@1 as brand, t_hour@2 as t_hour, t_minute@3 as t_minute, ext_price@4 as ext_price] + │ SortPreservingMergeExec: [ext_price@4 DESC, i_brand_id@5 ASC, t_hour@2 ASC] + │ SortExec: expr=[ext_price@4 DESC, brand_id@0 ASC, t_hour@2 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_brand_id@1 as brand_id, i_brand@0 as brand, t_hour@2 as t_hour, t_minute@3 as t_minute, sum(tmp.ext_price)@4 as ext_price, i_brand_id@1 as i_brand_id] + │ AggregateExec: mode=FinalPartitioned, gby=[i_brand@0 as i_brand, i_brand_id@1 as i_brand_id, t_hour@2 as t_hour, t_minute@3 as t_minute], aggr=[sum(tmp.ext_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_brand@0, i_brand_id@1, t_hour@2, t_minute@3], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_brand@1 as i_brand, i_brand_id@0 as i_brand_id, t_hour@3 as t_hour, t_minute@4 as t_minute], aggr=[sum(tmp.ext_price)] + │ ProjectionExec: expr=[i_brand_id@2 as i_brand_id, i_brand@3 as i_brand, ext_price@4 as ext_price, t_hour@0 as t_hour, t_minute@1 as t_minute] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@3, time_sk@3)], projection=[t_hour@1, t_minute@2, i_brand_id@4, i_brand@5, ext_price@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[i_brand_id@2 as i_brand_id, i_brand@3 as i_brand, ext_price@0 as ext_price, time_sk@1 as time_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(sold_item_sk@1, i_item_sk@0)], projection=[ext_price@0, time_sk@2, i_brand_id@4, i_brand@5] + │ CoalescePartitionsExec + │ UnionExec + │ ProjectionExec: expr=[ws_ext_sales_price@2 as ext_price, ws_item_sk@1 as sold_item_sk, ws_sold_time_sk@0 as time_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_sold_time_sk@3, ws_item_sk@4, ws_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_sold_time_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[cs_ext_sales_price@2 as ext_price, cs_item_sk@1 as sold_item_sk, cs_sold_time_sk@0 as time_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_sold_time_sk@3, cs_item_sk@4, cs_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_sold_time_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ss_ext_sales_price@2 as ext_price, ss_item_sk@1 as sold_item_sk, ss_sold_time_sk@0 as time_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_sold_time_sk@3, ss_item_sk@4, ss_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_sold_time_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_manager_id@3 = 1, projection=[i_item_sk@0, i_brand_id@1, i_brand@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@3 = 1 AND DynamicFilter [ empty ], pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, t_hour@1 as t_hour, t_minute@2 as t_minute, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: t_meal_time@3 = breakfast OR t_meal_time@3 = dinner, projection=[t_time_sk@0, t_hour@1, t_minute@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute, t_meal_time], file_type=parquet, predicate=t_meal_time@3 = breakfast OR t_meal_time@3 = dinner, pruning_predicate=t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= breakfast AND breakfast <= t_meal_time_max@1 OR t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= dinner AND dinner <= t_meal_time_max@1, required_guarantees=[t_meal_time in (breakfast, dinner)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1999, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 11 AND d_year@1 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1999, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 11 AND d_year@1 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1999, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 11 AND d_year@1 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_72() -> Result<()> { + let display = test_tpcds_query(72).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [total_cnt@5 DESC, i_item_desc@0 ASC, w_warehouse_name@1 ASC, d_week_seq@2 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[total_cnt@5 DESC, i_item_desc@0 ASC, w_warehouse_name@1 ASC, d_week_seq@2 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_desc@0 as i_item_desc, w_warehouse_name@1 as w_warehouse_name, d_week_seq@2 as d_week_seq, sum(CASE WHEN promotion.p_promo_sk IS NULL THEN Int64(1) ELSE Int64(0) END)@3 as no_promo, sum(CASE WHEN promotion.p_promo_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END)@4 as promo, count(Int64(1))@5 as total_cnt] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_desc@0 as i_item_desc, w_warehouse_name@1 as w_warehouse_name, d_week_seq@2 as d_week_seq], aggr=[sum(CASE WHEN promotion.p_promo_sk IS NULL THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN promotion.p_promo_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END), count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_desc@0, w_warehouse_name@1, d_week_seq@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_desc@1 as i_item_desc, w_warehouse_name@0 as w_warehouse_name, d_week_seq@2 as d_week_seq], aggr=[sum(CASE WHEN promotion.p_promo_sk IS NULL THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN promotion.p_promo_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END), count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(cs_item_sk@0, cr_item_sk@0), (cs_order_number@1, cr_order_number@1)], projection=[w_warehouse_name@2, i_item_desc@3, d_week_seq@4, p_promo_sk@5] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[cs_item_sk@1 as cs_item_sk, cs_order_number@2 as cs_order_number, w_warehouse_name@3 as w_warehouse_name, i_item_desc@4 as i_item_desc, d_week_seq@5 as d_week_seq, p_promo_sk@0 as p_promo_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(CAST(promotion.p_promo_sk AS Float64)@1, cs_promo_sk@1)], projection=[p_promo_sk@0, cs_item_sk@2, cs_order_number@4, w_warehouse_name@5, i_item_desc@6, d_week_seq@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_ship_date_sk@0, CAST(d3.d_date_sk AS Float64)@2)], filter=d_date@1 > d_date@0 + IntervalMonthDayNano { months: 0, days: 5, nanoseconds: 0 }, projection=[cs_item_sk@1, cs_promo_sk@2, cs_order_number@3, w_warehouse_name@4, i_item_desc@5, d_week_seq@7] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(inv_date_sk@4, d_date_sk@0), (d_week_seq@8, d_week_seq@1)], projection=[cs_ship_date_sk@0, cs_item_sk@1, cs_promo_sk@2, cs_order_number@3, w_warehouse_name@5, i_item_desc@6, d_date@7, d_week_seq@8] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[cs_ship_date_sk@2 as cs_ship_date_sk, cs_item_sk@3 as cs_item_sk, cs_promo_sk@4 as cs_promo_sk, cs_order_number@5 as cs_order_number, inv_date_sk@6 as inv_date_sk, w_warehouse_name@7 as w_warehouse_name, i_item_desc@8 as i_item_desc, d_date@0 as d_date, d_week_seq@1 as d_week_seq] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@3, cs_sold_date_sk@0)], projection=[d_date@1, d_week_seq@2, cs_ship_date_sk@5, cs_item_sk@6, cs_promo_sk@7, cs_order_number@8, inv_date_sk@9, w_warehouse_name@10, i_item_desc@11] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, cs_bill_hdemo_sk@2)], projection=[cs_sold_date_sk@2, cs_ship_date_sk@3, cs_item_sk@5, cs_promo_sk@6, cs_order_number@7, inv_date_sk@8, w_warehouse_name@9, i_item_desc@10] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, cs_bill_cdemo_sk@2)], projection=[cs_sold_date_sk@2, cs_ship_date_sk@3, cs_bill_hdemo_sk@5, cs_item_sk@6, cs_promo_sk@7, cs_order_number@8, inv_date_sk@9, w_warehouse_name@10, i_item_desc@11] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_bill_cdemo_sk@2], 3), input_partitions=3 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_ship_date_sk@2 as cs_ship_date_sk, cs_bill_cdemo_sk@3 as cs_bill_cdemo_sk, cs_bill_hdemo_sk@4 as cs_bill_hdemo_sk, cs_item_sk@5 as cs_item_sk, cs_promo_sk@6 as cs_promo_sk, cs_order_number@7 as cs_order_number, inv_date_sk@8 as inv_date_sk, w_warehouse_name@9 as w_warehouse_name, i_item_desc@0 as i_item_desc] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@4)], projection=[i_item_desc@1, cs_sold_date_sk@2, cs_ship_date_sk@3, cs_bill_cdemo_sk@4, cs_bill_hdemo_sk@5, cs_item_sk@6, cs_promo_sk@7, cs_order_number@8, inv_date_sk@9, w_warehouse_name@10] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_ship_date_sk@2 as cs_ship_date_sk, cs_bill_cdemo_sk@3 as cs_bill_cdemo_sk, cs_bill_hdemo_sk@4 as cs_bill_hdemo_sk, cs_item_sk@5 as cs_item_sk, cs_promo_sk@6 as cs_promo_sk, cs_order_number@7 as cs_order_number, inv_date_sk@8 as inv_date_sk, w_warehouse_name@0 as w_warehouse_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(w_warehouse_sk@0, inv_warehouse_sk@8)], projection=[w_warehouse_name@1, cs_sold_date_sk@2, cs_ship_date_sk@3, cs_bill_cdemo_sk@4, cs_bill_hdemo_sk@5, cs_item_sk@6, cs_promo_sk@7, cs_order_number@8, inv_date_sk@9] + │ CoalescePartitionsExec + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/warehouse/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet]]}, projection=[w_warehouse_sk, w_warehouse_name], file_type=parquet + │ ProjectionExec: expr=[cs_sold_date_sk@2 as cs_sold_date_sk, cs_ship_date_sk@3 as cs_ship_date_sk, cs_bill_cdemo_sk@4 as cs_bill_cdemo_sk, cs_bill_hdemo_sk@5 as cs_bill_hdemo_sk, cs_item_sk@6 as cs_item_sk, cs_promo_sk@7 as cs_promo_sk, cs_order_number@8 as cs_order_number, inv_date_sk@0 as inv_date_sk, inv_warehouse_sk@1 as inv_warehouse_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(inv_item_sk@1, cs_item_sk@4)], filter=inv_quantity_on_hand@1 < cs_quantity@0, projection=[inv_date_sk@0, inv_warehouse_sk@2, cs_sold_date_sk@4, cs_ship_date_sk@5, cs_bill_cdemo_sk@6, cs_bill_hdemo_sk@7, cs_item_sk@8, cs_promo_sk@9, cs_order_number@10] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, d_week_seq@2 as d_week_seq, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@3 = 1999, projection=[d_date_sk@0, d_date@1, d_week_seq@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_week_seq, d_year], file_type=parquet, predicate=d_year@3 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: hd_buy_potential@1 = >10000, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_buy_potential], file_type=parquet, predicate=hd_buy_potential@1 = >10000, pruning_predicate=hd_buy_potential_null_count@2 != row_count@3 AND hd_buy_potential_min@0 <= >10000 AND >10000 <= hd_buy_potential_max@1, required_guarantees=[hd_buy_potential in (>10000)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=3 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cd_marital_status@1 = D, projection=[cd_demo_sk@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=cd_marital_status@1 = D, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= D AND D <= cd_marital_status_max@1, required_guarantees=[cd_marital_status in (D)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_desc], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([inv_item_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], ...]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@4], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_bill_cdemo_sk, cs_bill_hdemo_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_73() -> Result<()> { + let display = test_tpcds_query(73).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [cnt@5 DESC, c_last_name@0 ASC NULLS LAST] + │ SortExec: expr=[cnt@5 DESC, c_last_name@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[c_last_name@4 as c_last_name, c_first_name@3 as c_first_name, c_salutation@2 as c_salutation, c_preferred_cust_flag@5 as c_preferred_cust_flag, ss_ticket_number@0 as ss_ticket_number, cnt@1 as cnt] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@1, CAST(customer.c_customer_sk AS Float64)@5)], projection=[ss_ticket_number@0, cnt@2, c_salutation@4, c_first_name@5, c_last_name@6, c_preferred_cust_flag@7] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, count(Int64(1))@2 as cnt] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: count(Int64(1))@2 >= 1 AND count(Int64(1))@2 <= 5 + │ AggregateExec: mode=FinalPartitioned, gby=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_ticket_number@0, ss_customer_sk@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_ticket_number@1 as ss_ticket_number, ss_customer_sk@0 as ss_customer_sk], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_customer_sk@2, ss_ticket_number@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@2)], projection=[ss_customer_sk@2, ss_hdemo_sk@3, ss_ticket_number@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_customer_sk@3, ss_hdemo_sk@4, ss_store_sk@5, ss_ticket_number@6] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_salutation@1 as c_salutation, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_salutation, c_first_name, c_last_name, c_preferred_cust_flag], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: (hd_buy_potential@1 = Unknown OR hd_buy_potential@1 = >10000) AND hd_vehicle_count@3 > 0 AND CASE WHEN hd_vehicle_count@3 > 0 THEN CAST(hd_dep_count@2 AS Float64) / CAST(hd_vehicle_count@3 AS Float64) END > 1, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_buy_potential, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=(hd_buy_potential@1 = Unknown OR hd_buy_potential@1 = >10000) AND hd_vehicle_count@3 > 0 AND CASE WHEN hd_vehicle_count@3 > 0 THEN CAST(hd_dep_count@2 AS Float64) / CAST(hd_vehicle_count@3 AS Float64) END > 1, pruning_predicate=(hd_buy_potential_null_count@2 != row_count@3 AND hd_buy_potential_min@0 <= Unknown AND Unknown <= hd_buy_potential_max@1 OR hd_buy_potential_null_count@2 != row_count@3 AND hd_buy_potential_min@0 <= >10000 AND >10000 <= hd_buy_potential_max@1) AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_max@4 > 0, required_guarantees=[hd_buy_potential in (>10000, Unknown)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_county@1 IN ([Orange County, Bronx County, Franklin Parish, Williamson County]), projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_county], file_type=parquet, predicate=s_county@1 IN ([Orange County, Bronx County, Franklin Parish, Williamson County]), pruning_predicate=s_county_null_count@2 != row_count@3 AND s_county_min@0 <= Orange County AND Orange County <= s_county_max@1 OR s_county_null_count@2 != row_count@3 AND s_county_min@0 <= Bronx County AND Bronx County <= s_county_max@1 OR s_county_null_count@2 != row_count@3 AND s_county_min@0 <= Franklin Parish AND Franklin Parish <= s_county_max@1 OR s_county_null_count@2 != row_count@3 AND s_county_min@0 <= Williamson County AND Williamson County <= s_county_max@1, required_guarantees=[s_county in (Bronx County, Franklin Parish, Orange County, Williamson County)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_dom@2 >= 1 AND d_dom@2 <= 2 AND (d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001), projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=d_dom@2 >= 1 AND d_dom@2 <= 2 AND (d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001), pruning_predicate=d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 2 AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_74() -> Result<()> { + let display = test_tpcds_query(74).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [customer_id@0 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[customer_id@0 ASC], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], filter=CASE WHEN year_total@2 > Some(0),17,2 THEN year_total@3 / year_total@2 END > CASE WHEN year_total@0 > Some(0),17,2 THEN year_total@1 / year_total@0 END, projection=[customer_id@2, customer_first_name@3, customer_last_name@4] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[customer_id@1 as customer_id, year_total@2 as year_total, customer_id@3 as customer_id, customer_first_name@4 as customer_first_name, customer_last_name@5 as customer_last_name, year_total@6 as year_total, year_total@0 as year_total] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], projection=[year_total@1, customer_id@2, year_total@3, customer_id@4, customer_first_name@5, customer_last_name@6, year_total@7] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_net_paid)@1 as year_total] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: sum(web_sales.ws_net_paid)@1 > Some(0),17,2 + │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(web_sales.ws_net_paid)@4 as sum(web_sales.ws_net_paid)] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, d_year@3 as d_year], aggr=[sum(web_sales.ws_net_paid)], ordering_mode=PartiallySorted([3]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, d_year@3], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, d_year@4 as d_year], aggr=[sum(web_sales.ws_net_paid)], ordering_mode=PartiallySorted([3]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, ws_net_paid@4 as ws_net_paid, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@3)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, ws_net_paid@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@4, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, ws_sold_date_sk@5, ws_net_paid@7] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(store_sales.ss_net_paid)@1 as year_total] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: sum(store_sales.ss_net_paid)@1 > Some(0),17,2 + │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(store_sales.ss_net_paid)@4 as sum(store_sales.ss_net_paid)] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, d_year@3 as d_year], aggr=[sum(store_sales.ss_net_paid)], ordering_mode=PartiallySorted([3]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, d_year@3], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, d_year@4 as d_year], aggr=[sum(store_sales.ss_net_paid)], ordering_mode=PartiallySorted([3]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, ss_net_paid@4 as ss_net_paid, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@3)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, ss_net_paid@7] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@4, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, ss_sold_date_sk@5, ss_net_paid@7] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, c_first_name@1 as customer_first_name, c_last_name@2 as customer_last_name, sum(store_sales.ss_net_paid)@4 as year_total] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, d_year@3 as d_year], aggr=[sum(store_sales.ss_net_paid)], ordering_mode=PartiallySorted([3]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, d_year@3], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, d_year@4 as d_year], aggr=[sum(store_sales.ss_net_paid)], ordering_mode=PartiallySorted([3]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, ss_net_paid@4 as ss_net_paid, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@3)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, ss_net_paid@7] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@4, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, ss_sold_date_sk@5, ss_net_paid@7] + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_net_paid)@4 as year_total] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, d_year@3 as d_year], aggr=[sum(web_sales.ws_net_paid)], ordering_mode=PartiallySorted([3]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, d_year@3], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, d_year@4 as d_year], aggr=[sum(web_sales.ws_net_paid)], ordering_mode=PartiallySorted([3]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, ws_net_paid@4 as ws_net_paid, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@3)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, ws_net_paid@7] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@4, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, ws_sold_date_sk@5, ws_net_paid@7] + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@4], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@4], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@4], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@4], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_75() -> Result<()> { + let display = test_tpcds_query(75).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [sales_cnt_diff@8 ASC NULLS LAST, sales_amt_diff@9 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[sales_cnt_diff@8 ASC NULLS LAST, sales_amt_diff@9 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[d_year@7 as prev_year, d_year@0 as year_, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sales_cnt@8 as prev_yr_cnt, sales_cnt@5 as curr_yr_cnt, sales_cnt@5 - sales_cnt@8 as sales_cnt_diff, sales_amt@6 - sales_amt@9 as sales_amt_diff] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_brand_id@1, i_brand_id@1), (i_class_id@2, i_class_id@2), (i_category_id@3, i_category_id@3), (i_manufact_id@4, i_manufact_id@4)], filter=CAST(sales_cnt@0 AS Decimal128(17, 2)) / CAST(sales_cnt@1 AS Decimal128(17, 2)) < Some(900000),23,6, projection=[d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, sales_cnt@5, sales_amt@6, d_year@7, sales_cnt@12, sales_amt@13] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sum(sales_detail.sales_cnt)@5 as sales_cnt, sum(sales_detail.sales_amt)@6 as sales_amt] + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id], aggr=[sum(sales_detail.sales_cnt), sum(sales_detail.sales_amt)], ordering_mode=PartiallySorted([0]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id], aggr=[sum(sales_detail.sales_cnt), sum(sales_detail.sales_amt)], ordering_mode=PartiallySorted([0]) + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sales_cnt@5 as sales_cnt, sales_amt@6 as sales_amt], aggr=[], ordering_mode=PartiallySorted([0]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, sales_cnt@5, sales_amt@6], 3), input_partitions=9 + │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sales_cnt@5 as sales_cnt, sales_amt@6 as sales_amt], aggr=[], ordering_mode=PartiallySorted([0]) + │ UnionExec + │ ProjectionExec: expr=[d_year@6 as d_year, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, cs_quantity@0 - coalesce(cr_return_quantity@7, 0) as sales_cnt, cs_ext_sales_price@1 - coalesce(CAST(cr_return_amount@8 AS Decimal128(30, 15)), Some(0),30,15) as sales_amt] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(cs_order_number@1, cr_order_number@1), (cs_item_sk@0, cr_item_sk@0)], projection=[cs_quantity@2, cs_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, cr_return_quantity@11, cr_return_amount@12] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[cs_item_sk@1 as cs_item_sk, cs_order_number@2 as cs_order_number, cs_quantity@3 as cs_quantity, cs_ext_sales_price@4 as cs_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@0)], projection=[d_year@1, cs_item_sk@4, cs_order_number@5, cs_quantity@6, cs_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@4 as cs_sold_date_sk, cs_item_sk@5 as cs_item_sk, cs_order_number@6 as cs_order_number, cs_quantity@7 as cs_quantity, cs_ext_sales_price@8 as cs_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, cs_sold_date_sk@5, cs_item_sk@6, cs_order_number@7, cs_quantity@8, cs_ext_sales_price@9] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet + │ ProjectionExec: expr=[d_year@6 as d_year, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, ss_quantity@0 - coalesce(sr_return_quantity@7, 0) as sales_cnt, ss_ext_sales_price@1 - coalesce(CAST(sr_return_amt@8 AS Decimal128(30, 15)), Some(0),30,15) as sales_amt] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(ss_ticket_number@1, sr_ticket_number@1), (ss_item_sk@0, sr_item_sk@0)], projection=[ss_quantity@2, ss_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, sr_return_quantity@11, sr_return_amt@12] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_ticket_number@2 as ss_ticket_number, ss_quantity@3 as ss_quantity, ss_ext_sales_price@4 as ss_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_ticket_number@5, ss_quantity@6, ss_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@4 as ss_sold_date_sk, ss_item_sk@5 as ss_item_sk, ss_ticket_number@6 as ss_ticket_number, ss_quantity@7 as ss_quantity, ss_ext_sales_price@8 as ss_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, ss_sold_date_sk@5, ss_item_sk@6, ss_ticket_number@7, ss_quantity@8, ss_ext_sales_price@9] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet + │ ProjectionExec: expr=[d_year@6 as d_year, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, ws_quantity@0 - coalesce(wr_return_quantity@7, 0) as sales_cnt, ws_ext_sales_price@1 - coalesce(CAST(wr_return_amt@8 AS Decimal128(30, 15)), Some(0),30,15) as sales_amt] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(ws_order_number@1, wr_order_number@1), (ws_item_sk@0, wr_item_sk@0)], projection=[ws_quantity@2, ws_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, wr_return_quantity@11, wr_return_amt@12] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ws_item_sk@1 as ws_item_sk, ws_order_number@2 as ws_order_number, ws_quantity@3 as ws_quantity, ws_ext_sales_price@4 as ws_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@0)], projection=[d_year@1, ws_item_sk@4, ws_order_number@5, ws_quantity@6, ws_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@4 as ws_sold_date_sk, ws_item_sk@5 as ws_item_sk, ws_order_number@6 as ws_order_number, ws_quantity@7 as ws_quantity, ws_ext_sales_price@8 as ws_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, ws_sold_date_sk@5, ws_item_sk@6, ws_order_number@7, ws_quantity@8, ws_ext_sales_price@9] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet + │ ProjectionExec: expr=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sum(sales_detail.sales_cnt)@5 as sales_cnt, sum(sales_detail.sales_amt)@6 as sales_amt] + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id], aggr=[sum(sales_detail.sales_cnt), sum(sales_detail.sales_amt)], ordering_mode=PartiallySorted([0]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id], aggr=[sum(sales_detail.sales_cnt), sum(sales_detail.sales_amt)], ordering_mode=PartiallySorted([0]) + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sales_cnt@5 as sales_cnt, sales_amt@6 as sales_amt], aggr=[], ordering_mode=PartiallySorted([0]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, sales_cnt@5, sales_amt@6], 3), input_partitions=9 + │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sales_cnt@5 as sales_cnt, sales_amt@6 as sales_amt], aggr=[], ordering_mode=PartiallySorted([0]) + │ UnionExec + │ ProjectionExec: expr=[d_year@6 as d_year, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, cs_quantity@0 - coalesce(cr_return_quantity@7, 0) as sales_cnt, cs_ext_sales_price@1 - coalesce(CAST(cr_return_amount@8 AS Decimal128(30, 15)), Some(0),30,15) as sales_amt] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(cs_order_number@1, cr_order_number@1), (cs_item_sk@0, cr_item_sk@0)], projection=[cs_quantity@2, cs_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, cr_return_quantity@11, cr_return_amount@12] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[cs_item_sk@1 as cs_item_sk, cs_order_number@2 as cs_order_number, cs_quantity@3 as cs_quantity, cs_ext_sales_price@4 as cs_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@0)], projection=[d_year@1, cs_item_sk@4, cs_order_number@5, cs_quantity@6, cs_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@4 as cs_sold_date_sk, cs_item_sk@5 as cs_item_sk, cs_order_number@6 as cs_order_number, cs_quantity@7 as cs_quantity, cs_ext_sales_price@8 as cs_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, cs_sold_date_sk@5, cs_item_sk@6, cs_order_number@7, cs_quantity@8, cs_ext_sales_price@9] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet + │ ProjectionExec: expr=[d_year@6 as d_year, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, ss_quantity@0 - coalesce(sr_return_quantity@7, 0) as sales_cnt, ss_ext_sales_price@1 - coalesce(CAST(sr_return_amt@8 AS Decimal128(30, 15)), Some(0),30,15) as sales_amt] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(ss_ticket_number@1, sr_ticket_number@1), (ss_item_sk@0, sr_item_sk@0)], projection=[ss_quantity@2, ss_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, sr_return_quantity@11, sr_return_amt@12] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_ticket_number@2 as ss_ticket_number, ss_quantity@3 as ss_quantity, ss_ext_sales_price@4 as ss_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_ticket_number@5, ss_quantity@6, ss_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@4 as ss_sold_date_sk, ss_item_sk@5 as ss_item_sk, ss_ticket_number@6 as ss_ticket_number, ss_quantity@7 as ss_quantity, ss_ext_sales_price@8 as ss_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, ss_sold_date_sk@5, ss_item_sk@6, ss_ticket_number@7, ss_quantity@8, ss_ext_sales_price@9] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet + │ ProjectionExec: expr=[d_year@6 as d_year, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, ws_quantity@0 - coalesce(wr_return_quantity@7, 0) as sales_cnt, ws_ext_sales_price@1 - coalesce(CAST(wr_return_amt@8 AS Decimal128(30, 15)), Some(0),30,15) as sales_amt] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(ws_order_number@1, wr_order_number@1), (ws_item_sk@0, wr_item_sk@0)], projection=[ws_quantity@2, ws_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, wr_return_quantity@11, wr_return_amt@12] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ws_item_sk@1 as ws_item_sk, ws_order_number@2 as ws_order_number, ws_quantity@3 as ws_quantity, ws_ext_sales_price@4 as ws_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@0)], projection=[d_year@1, ws_item_sk@4, ws_order_number@5, ws_quantity@6, ws_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@4 as ws_sold_date_sk, ws_item_sk@5 as ws_item_sk, ws_order_number@6 as ws_order_number, ws_quantity@7 as ws_quantity, ws_ext_sales_price@8 as ws_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, ws_sold_date_sk@5, ws_item_sk@6, ws_order_number@7, ws_quantity@8, ws_ext_sales_price@9] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@4 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@4 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@4 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@4 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@4 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@4 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_76() -> Result<()> { + let display = test_tpcds_query(76).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [channel@0 ASC, col_name@1 ASC, d_year@2 ASC, d_qoy@3 ASC, i_category@4 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, col_name@1 ASC, d_year@2 ASC, d_qoy@3 ASC, i_category@4 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[channel@0 as channel, col_name@1 as col_name, d_year@2 as d_year, d_qoy@3 as d_qoy, i_category@4 as i_category, count(Int64(1))@5 as sales_cnt, sum(foo.ext_sales_price)@6 as sales_amt] + │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, col_name@1 as col_name, d_year@2 as d_year, d_qoy@3 as d_qoy, i_category@4 as i_category], aggr=[count(Int64(1)), sum(foo.ext_sales_price)] + │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([channel@0, col_name@1, d_year@2, d_qoy@3, i_category@4], 3), input_partitions=9 + │ AggregateExec: mode=Partial, gby=[channel@0 as channel, col_name@1 as col_name, d_year@2 as d_year, d_qoy@3 as d_qoy, i_category@4 as i_category], aggr=[count(Int64(1)), sum(foo.ext_sales_price)], ordering_mode=PartiallySorted([0, 1]) + │ UnionExec + │ ProjectionExec: expr=[store as channel, ss_store_sk as col_name, d_year@4 as d_year, d_qoy@5 as d_qoy, i_category@2 as i_category, ss_ext_sales_price@1 as ext_sales_price] + │ ProjectionExec: expr=[ss_sold_date_sk@3 as ss_sold_date_sk, ss_ext_sales_price@4 as ss_ext_sales_price, i_category@5 as i_category, d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@0)], projection=[d_date_sk@0, d_year@1, d_qoy@2, ss_sold_date_sk@4, ss_ext_sales_price@5, i_category@6] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[web as channel, ws_ship_customer_sk as col_name, d_year@4 as d_year, d_qoy@5 as d_qoy, i_category@2 as i_category, ws_ext_sales_price@1 as ext_sales_price] + │ ProjectionExec: expr=[ws_sold_date_sk@3 as ws_sold_date_sk, ws_ext_sales_price@4 as ws_ext_sales_price, i_category@5 as i_category, d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ws_sold_date_sk@0)], projection=[d_date_sk@0, d_year@1, d_qoy@2, ws_sold_date_sk@4, ws_ext_sales_price@5, i_category@6] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[catalog as channel, cs_ship_addr_sk as col_name, d_year@4 as d_year, d_qoy@5 as d_qoy, i_category@2 as i_category, cs_ext_sales_price@1 as ext_sales_price] + │ ProjectionExec: expr=[cs_sold_date_sk@3 as cs_sold_date_sk, cs_ext_sales_price@4 as cs_ext_sales_price, i_category@5 as i_category, d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, cs_sold_date_sk@0)], projection=[d_date_sk@0, d_year@1, d_qoy@2, cs_sold_date_sk@4, cs_ext_sales_price@5, i_category@6] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 6), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_sold_date_sk@0], 6), input_partitions=3 + │ ProjectionExec: expr=[ss_sold_date_sk@1 as ss_sold_date_sk, ss_ext_sales_price@2 as ss_ext_sales_price, i_category@0 as i_category] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_category@1, ss_sold_date_sk@2, ss_ext_sales_price@4] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 6), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_store_sk@2 IS NULL, projection=[ss_sold_date_sk@0, ss_item_sk@1, ss_ext_sales_price@3] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=ss_store_sk@2 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=ss_store_sk_null_count@0 > 0, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 6), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_sold_date_sk@0], 6), input_partitions=3 + │ ProjectionExec: expr=[ws_sold_date_sk@1 as ws_sold_date_sk, ws_ext_sales_price@2 as ws_ext_sales_price, i_category@0 as i_category] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_category@1, ws_sold_date_sk@2, ws_ext_sales_price@4] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 6), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ws_ship_customer_sk@2 IS NULL, projection=[ws_sold_date_sk@0, ws_item_sk@1, ws_ext_sales_price@3] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ship_customer_sk, ws_ext_sales_price], file_type=parquet, predicate=ws_ship_customer_sk@2 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=ws_ship_customer_sk_null_count@0 > 0, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 6), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_sold_date_sk@0], 6), input_partitions=3 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_ext_sales_price@2 as cs_ext_sales_price, i_category@0 as i_category] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_category@1, cs_sold_date_sk@2, cs_ext_sales_price@4] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 6), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cs_ship_addr_sk@1 IS NULL, projection=[cs_sold_date_sk@0, cs_item_sk@2, cs_ext_sales_price@3] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_ship_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=cs_ship_addr_sk@1 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=cs_ship_addr_sk_null_count@0 > 0, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_77() -> Result<()> { + let display = test_tpcds_query(77).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [channel@0 ASC, id@1 ASC, returns_@3 DESC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, id@1 ASC, returns_@3 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[channel@0 as channel, id@1 as id, sum(x.sales)@3 as sales, sum(x.returns_)@4 as returns_, sum(x.profit)@5 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, id@1 as id, __grouping_id@2 as __grouping_id], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([channel@0, id@1, __grouping_id@2], 3), input_partitions=9 + │ AggregateExec: mode=Partial, gby=[(NULL as channel, NULL as id), (channel@0 as channel, NULL as id), (channel@0 as channel, id@1 as id)], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] + │ UnionExec + │ ProjectionExec: expr=[store channel as channel, CAST(s_store_sk@0 AS Float64) as id, sales@1 as sales, coalesce(CAST(returns_@4 AS Decimal128(22, 2)), Some(0),22,2) as returns_, profit@2 - coalesce(CAST(profit_loss@5 AS Decimal128(22, 2)), Some(0),22,2) as profit] + │ ProjectionExec: expr=[s_store_sk@3 as s_store_sk, sales@4 as sales, profit@5 as profit, s_store_sk@0 as s_store_sk, returns_@1 as returns_, profit_loss@2 as profit_loss] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(s_store_sk@0, s_store_sk@0)] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, sum(store_returns.sr_return_amt)@1 as returns_, sum(store_returns.sr_net_loss)@2 as profit_loss] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_sk@0 as s_store_sk], aggr=[sum(store_returns.sr_return_amt), sum(store_returns.sr_net_loss)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([s_store_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[s_store_sk@2 as s_store_sk], aggr=[sum(store_returns.sr_return_amt), sum(store_returns.sr_net_loss)] + │ ProjectionExec: expr=[sr_return_amt@1 as sr_return_amt, sr_net_loss@2 as sr_net_loss, s_store_sk@0 as s_store_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, sr_store_sk@0)], projection=[s_store_sk@0, sr_return_amt@3, sr_net_loss@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, sr_returned_date_sk@0)], projection=[sr_store_sk@3, sr_return_amt@4, sr_net_loss@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_store_sk, sr_return_amt, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, sum(store_sales.ss_ext_sales_price)@1 as sales, sum(store_sales.ss_net_profit)@2 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_sk@0 as s_store_sk], aggr=[sum(store_sales.ss_ext_sales_price), sum(store_sales.ss_net_profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([s_store_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[s_store_sk@2 as s_store_sk], aggr=[sum(store_sales.ss_ext_sales_price), sum(store_sales.ss_net_profit)] + │ ProjectionExec: expr=[ss_ext_sales_price@1 as ss_ext_sales_price, ss_net_profit@2 as ss_net_profit, s_store_sk@0 as s_store_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)], projection=[s_store_sk@0, ss_ext_sales_price@3, ss_net_profit@4] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_store_sk@3, ss_ext_sales_price@4, ss_net_profit@5] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[catalog channel as channel, cs_call_center_sk@0 as id, sales@1 as sales, CAST(returns_@3 AS Decimal128(22, 2)) as returns_, CAST(profit@2 - profit_loss@4 AS Decimal128(23, 2)) as profit] + │ CrossJoinExec + │ CoalescePartitionsExec + │ ProjectionExec: expr=[cs_call_center_sk@0 as cs_call_center_sk, sum(catalog_sales.cs_ext_sales_price)@1 as sales, sum(catalog_sales.cs_net_profit)@2 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[cs_call_center_sk@0 as cs_call_center_sk], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(catalog_sales.cs_net_profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_call_center_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cs_call_center_sk@0 as cs_call_center_sk], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(catalog_sales.cs_net_profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_call_center_sk@3, cs_ext_sales_price@4, cs_net_profit@5] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_ext_sales_price, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[sum(catalog_returns.cr_return_amount)@1 as returns_, sum(catalog_returns.cr_net_loss)@2 as profit_loss] + │ AggregateExec: mode=FinalPartitioned, gby=[cr_call_center_sk@0 as cr_call_center_sk], aggr=[sum(catalog_returns.cr_return_amount), sum(catalog_returns.cr_net_loss)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cr_call_center_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cr_call_center_sk@0 as cr_call_center_sk], aggr=[sum(catalog_returns.cr_return_amount), sum(catalog_returns.cr_net_loss)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cr_returned_date_sk@0)], projection=[cr_call_center_sk@2, cr_return_amount@3, cr_net_loss@4] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_call_center_sk, cr_return_amount, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[web channel as channel, CAST(wp_web_page_sk@0 AS Float64) as id, sales@1 as sales, coalesce(CAST(returns_@4 AS Decimal128(22, 2)), Some(0),22,2) as returns_, profit@2 - coalesce(CAST(profit_loss@5 AS Decimal128(22, 2)), Some(0),22,2) as profit] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(wp_web_page_sk@0, wp_web_page_sk@0)] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[wp_web_page_sk@0 as wp_web_page_sk, sum(web_sales.ws_ext_sales_price)@1 as sales, sum(web_sales.ws_net_profit)@2 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[wp_web_page_sk@0 as wp_web_page_sk], aggr=[sum(web_sales.ws_ext_sales_price), sum(web_sales.ws_net_profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([wp_web_page_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[wp_web_page_sk@2 as wp_web_page_sk], aggr=[sum(web_sales.ws_ext_sales_price), sum(web_sales.ws_net_profit)] + │ ProjectionExec: expr=[ws_ext_sales_price@1 as ws_ext_sales_price, ws_net_profit@2 as ws_net_profit, wp_web_page_sk@0 as wp_web_page_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_page.wp_web_page_sk AS Float64)@1, ws_web_page_sk@0)], projection=[wp_web_page_sk@0, ws_ext_sales_price@3, ws_net_profit@4] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_web_page_sk@3, ws_ext_sales_price@4, ws_net_profit@5] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_web_page_sk, ws_ext_sales_price, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[wp_web_page_sk@0 as wp_web_page_sk, sum(web_returns.wr_return_amt)@1 as returns_, sum(web_returns.wr_net_loss)@2 as profit_loss] + │ AggregateExec: mode=FinalPartitioned, gby=[wp_web_page_sk@0 as wp_web_page_sk], aggr=[sum(web_returns.wr_return_amt), sum(web_returns.wr_net_loss)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([wp_web_page_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[wp_web_page_sk@2 as wp_web_page_sk], aggr=[sum(web_returns.wr_return_amt), sum(web_returns.wr_net_loss)] + │ ProjectionExec: expr=[wr_return_amt@1 as wr_return_amt, wr_net_loss@2 as wr_net_loss, wp_web_page_sk@0 as wp_web_page_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_page.wp_web_page_sk AS Float64)@1, wr_web_page_sk@0)], projection=[wp_web_page_sk@0, wr_return_amt@3, wr_net_loss@4] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, wr_returned_date_sk@0)], projection=[wr_web_page_sk@3, wr_return_amt@4, wr_net_loss@5] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_returned_date_sk, wr_web_page_sk, wr_return_amt, wr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[wp_web_page_sk@0 as wp_web_page_sk, CAST(wp_web_page_sk@0 AS Float64) as CAST(web_page.wp_web_page_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-3.parquet]]}, projection=[wp_web_page_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[wp_web_page_sk@0 as wp_web_page_sk, CAST(wp_web_page_sk@0 AS Float64) as CAST(web_page.wp_web_page_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-3.parquet]]}, projection=[wp_web_page_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_78() -> Result<()> { + let display = test_tpcds_query(78).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[ss_sold_year@0 as ss_sold_year, ss_item_sk@1 as ss_item_sk, ss_customer_sk@2 as ss_customer_sk, ratio@3 as ratio, store_qty@4 as store_qty, store_wholesale_cost@5 as store_wholesale_cost, store_sales_price@6 as store_sales_price, other_chan_qty@7 as other_chan_qty, other_chan_wholesale_cost@8 as other_chan_wholesale_cost, other_chan_sales_price@9 as other_chan_sales_price] + │ SortPreservingMergeExec: [ss_sold_year@0 ASC NULLS LAST, ss_item_sk@1 ASC NULLS LAST, ss_customer_sk@2 ASC NULLS LAST, ss_qty@10 DESC, ss_wc@11 DESC, ss_sp@12 DESC, other_chan_qty@7 ASC NULLS LAST, other_chan_wholesale_cost@8 ASC NULLS LAST, other_chan_sales_price@9 ASC NULLS LAST, ratio@3 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[ss_item_sk@1 ASC NULLS LAST, ss_customer_sk@2 ASC NULLS LAST, store_qty@4 DESC, store_wholesale_cost@5 DESC, store_sales_price@6 DESC, other_chan_qty@7 ASC NULLS LAST, other_chan_wholesale_cost@8 ASC NULLS LAST, other_chan_sales_price@9 ASC NULLS LAST, ratio@3 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[ss_sold_year@1 as ss_sold_year, ss_item_sk@2 as ss_item_sk, ss_customer_sk@3 as ss_customer_sk, round(ss_qty@4 / __common_expr_1@0, 2) as ratio, ss_qty@4 as store_qty, ss_wc@5 as store_wholesale_cost, ss_sp@6 as store_sales_price, __common_expr_1@0 as other_chan_qty, coalesce(CAST(ws_wc@7 AS Decimal128(22, 2)), Some(0),22,2) + coalesce(CAST(cs_wc@9 AS Decimal128(22, 2)), Some(0),22,2) as other_chan_wholesale_cost, coalesce(CAST(ws_sp@8 AS Decimal128(22, 2)), Some(0),22,2) + coalesce(CAST(cs_sp@10 AS Decimal128(22, 2)), Some(0),22,2) as other_chan_sales_price, ss_qty@4 as ss_qty, ss_wc@5 as ss_wc, ss_sp@6 as ss_sp] + │ ProjectionExec: expr=[coalesce(ws_qty@6, 0) + coalesce(cs_qty@9, 0) as __common_expr_1, ss_sold_year@0 as ss_sold_year, ss_item_sk@1 as ss_item_sk, ss_customer_sk@2 as ss_customer_sk, ss_qty@3 as ss_qty, ss_wc@4 as ss_wc, ss_sp@5 as ss_sp, ws_wc@7 as ws_wc, ws_sp@8 as ws_sp, cs_wc@10 as cs_wc, cs_sp@11 as cs_sp] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: coalesce(ws_qty@6, 0) > 0 OR coalesce(cs_qty@9, 0) > 0 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(ss_sold_year@0, cs_sold_year@0), (ss_item_sk@1, cs_item_sk@1), (ss_customer_sk@2, cs_customer_sk@2)], projection=[ss_sold_year@0, ss_item_sk@1, ss_customer_sk@2, ss_qty@3, ss_wc@4, ss_sp@5, ws_qty@6, ws_wc@7, ws_sp@8, cs_qty@12, cs_wc@13, cs_sp@14] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(ss_sold_year@0, ws_sold_year@0), (ss_item_sk@1, ws_item_sk@1), (ss_customer_sk@2, ws_customer_sk@2)], projection=[ss_sold_year@0, ss_item_sk@1, ss_customer_sk@2, ss_qty@3, ss_wc@4, ss_sp@5, ws_qty@9, ws_wc@10, ws_sp@11] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[d_year@0 as ss_sold_year, ss_item_sk@1 as ss_item_sk, ss_customer_sk@2 as ss_customer_sk, sum(store_sales.ss_quantity)@3 as ss_qty, sum(store_sales.ss_wholesale_cost)@4 as ss_wc, sum(store_sales.ss_sales_price)@5 as ss_sp] + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, ss_item_sk@1 as ss_item_sk, ss_customer_sk@2 as ss_customer_sk], aggr=[sum(store_sales.ss_quantity), sum(store_sales.ss_wholesale_cost), sum(store_sales.ss_sales_price)], ordering_mode=PartiallySorted([0]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([d_year@0, ss_item_sk@1, ss_customer_sk@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@5 as d_year, ss_item_sk@0 as ss_item_sk, ss_customer_sk@1 as ss_customer_sk], aggr=[sum(store_sales.ss_quantity), sum(store_sales.ss_wholesale_cost), sum(store_sales.ss_sales_price)], ordering_mode=PartiallySorted([0]) + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_customer_sk@2 as ss_customer_sk, ss_quantity@3 as ss_quantity, ss_wholesale_cost@4 as ss_wholesale_cost, ss_sales_price@5 as ss_sales_price, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_customer_sk@5, ss_quantity@6, ss_wholesale_cost@7, ss_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: sr_ticket_number@6 IS NULL, projection=[ss_sold_date_sk@0, ss_item_sk@1, ss_customer_sk@2, ss_quantity@3, ss_wholesale_cost@4, ss_sales_price@5] + │ ProjectionExec: expr=[ss_sold_date_sk@1 as ss_sold_date_sk, ss_item_sk@2 as ss_item_sk, ss_customer_sk@3 as ss_customer_sk, ss_quantity@4 as ss_quantity, ss_wholesale_cost@5 as ss_wholesale_cost, ss_sales_price@6 as ss_sales_price, sr_ticket_number@0 as sr_ticket_number] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(sr_ticket_number@1, ss_ticket_number@3), (sr_item_sk@0, ss_item_sk@1)], projection=[sr_ticket_number@1, ss_sold_date_sk@2, ss_item_sk@3, ss_customer_sk@4, ss_quantity@6, ss_wholesale_cost@7, ss_sales_price@8] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[d_year@0 as ws_sold_year, ws_item_sk@1 as ws_item_sk, ws_bill_customer_sk@2 as ws_customer_sk, sum(web_sales.ws_quantity)@3 as ws_qty, sum(web_sales.ws_wholesale_cost)@4 as ws_wc, sum(web_sales.ws_sales_price)@5 as ws_sp] + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, ws_item_sk@1 as ws_item_sk, ws_bill_customer_sk@2 as ws_bill_customer_sk], aggr=[sum(web_sales.ws_quantity), sum(web_sales.ws_wholesale_cost), sum(web_sales.ws_sales_price)] + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[d_year@0 as cs_sold_year, cs_item_sk@1 as cs_item_sk, cs_bill_customer_sk@2 as cs_customer_sk, sum(catalog_sales.cs_quantity)@3 as cs_qty, sum(catalog_sales.cs_wholesale_cost)@4 as cs_wc, sum(catalog_sales.cs_sales_price)@5 as cs_sp] + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, cs_item_sk@1 as cs_item_sk, cs_bill_customer_sk@2 as cs_bill_customer_sk], aggr=[sum(catalog_sales.cs_quantity), sum(catalog_sales.cs_wholesale_cost), sum(catalog_sales.cs_sales_price)] + │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2000 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_ticket_number@3, ss_item_sk@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_sales_price], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([d_year@0, ws_item_sk@1, ws_bill_customer_sk@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@5 as d_year, ws_item_sk@0 as ws_item_sk, ws_bill_customer_sk@1 as ws_bill_customer_sk], aggr=[sum(web_sales.ws_quantity), sum(web_sales.ws_wholesale_cost), sum(web_sales.ws_sales_price)] + │ ProjectionExec: expr=[ws_item_sk@1 as ws_item_sk, ws_bill_customer_sk@2 as ws_bill_customer_sk, ws_quantity@3 as ws_quantity, ws_wholesale_cost@4 as ws_wholesale_cost, ws_sales_price@5 as ws_sales_price, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@0)], projection=[d_year@1, ws_item_sk@4, ws_bill_customer_sk@5, ws_quantity@6, ws_wholesale_cost@7, ws_sales_price@8] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@2], 6), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_sold_date_sk@0], 6), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: wr_order_number@6 IS NULL, projection=[ws_sold_date_sk@0, ws_item_sk@1, ws_bill_customer_sk@2, ws_quantity@3, ws_wholesale_cost@4, ws_sales_price@5] + │ ProjectionExec: expr=[ws_sold_date_sk@1 as ws_sold_date_sk, ws_item_sk@2 as ws_item_sk, ws_bill_customer_sk@3 as ws_bill_customer_sk, ws_quantity@4 as ws_quantity, ws_wholesale_cost@5 as ws_wholesale_cost, ws_sales_price@6 as ws_sales_price, wr_order_number@0 as wr_order_number] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(wr_order_number@1, ws_order_number@3), (wr_item_sk@0, ws_item_sk@1)], projection=[wr_order_number@1, ws_sold_date_sk@2, ws_item_sk@3, ws_bill_customer_sk@4, ws_quantity@6, ws_wholesale_cost@7, ws_sales_price@8] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([wr_order_number@1, wr_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_order_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_order_number@3, ws_item_sk@1], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_order_number, ws_quantity, ws_wholesale_cost, ws_sales_price], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([d_year@0, cs_item_sk@1, cs_bill_customer_sk@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@5 as d_year, cs_item_sk@1 as cs_item_sk, cs_bill_customer_sk@0 as cs_bill_customer_sk], aggr=[sum(catalog_sales.cs_quantity), sum(catalog_sales.cs_wholesale_cost), sum(catalog_sales.cs_sales_price)] + │ ProjectionExec: expr=[cs_bill_customer_sk@1 as cs_bill_customer_sk, cs_item_sk@2 as cs_item_sk, cs_quantity@3 as cs_quantity, cs_wholesale_cost@4 as cs_wholesale_cost, cs_sales_price@5 as cs_sales_price, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@0)], projection=[d_year@1, cs_bill_customer_sk@4, cs_item_sk@5, cs_quantity@6, cs_wholesale_cost@7, cs_sales_price@8] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@2], 6), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_sold_date_sk@0], 6), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cr_order_number@6 IS NULL, projection=[cs_sold_date_sk@0, cs_bill_customer_sk@1, cs_item_sk@2, cs_quantity@3, cs_wholesale_cost@4, cs_sales_price@5] + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_bill_customer_sk@2 as cs_bill_customer_sk, cs_item_sk@3 as cs_item_sk, cs_quantity@4 as cs_quantity, cs_wholesale_cost@5 as cs_wholesale_cost, cs_sales_price@6 as cs_sales_price, cr_order_number@0 as cr_order_number] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(cr_order_number@1, cs_order_number@3), (cr_item_sk@0, cs_item_sk@2)], projection=[cr_order_number@1, cs_sold_date_sk@2, cs_bill_customer_sk@3, cs_item_sk@4, cs_quantity@6, cs_wholesale_cost@7, cs_sales_price@8] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cr_order_number@1, cr_item_sk@0], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_order_number@3, cs_item_sk@2], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_order_number, cs_quantity, cs_wholesale_cost, cs_sales_price], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_79() -> Result<()> { + let display = test_tpcds_query(79).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c_last_name@0 ASC, c_first_name@1 ASC, substr(ms.s_city,Int64(1),Int64(30))@2 ASC, profit@5 ASC, ss_ticket_number@3 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[c_last_name@0 ASC, c_first_name@1 ASC, substr(ms.s_city,Int64(1),Int64(30))@2 ASC, profit@5 ASC, ss_ticket_number@3 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[c_last_name@5 as c_last_name, c_first_name@4 as c_first_name, substr(s_city@1, 1, 30) as substr(ms.s_city,Int64(1),Int64(30)), ss_ticket_number@0 as ss_ticket_number, amt@2 as amt, profit@3 as profit] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@1, CAST(customer.c_customer_sk AS Float64)@3)], projection=[ss_ticket_number@0, s_city@2, amt@3, profit@4, c_first_name@6, c_last_name@7] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, s_city@3 as s_city, sum(store_sales.ss_coupon_amt)@4 as amt, sum(store_sales.ss_net_profit)@5 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, ss_addr_sk@2 as ss_addr_sk, s_city@3 as s_city], aggr=[sum(store_sales.ss_coupon_amt), sum(store_sales.ss_net_profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_ticket_number@0, ss_customer_sk@1, ss_addr_sk@2, s_city@3], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_ticket_number@2 as ss_ticket_number, ss_customer_sk@0 as ss_customer_sk, ss_addr_sk@1 as ss_addr_sk, s_city@5 as s_city], aggr=[sum(store_sales.ss_coupon_amt), sum(store_sales.ss_net_profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_customer_sk@2, ss_addr_sk@4, ss_ticket_number@5, ss_coupon_amt@6, ss_net_profit@7, s_city@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_customer_sk@1 as ss_customer_sk, ss_hdemo_sk@2 as ss_hdemo_sk, ss_addr_sk@3 as ss_addr_sk, ss_ticket_number@4 as ss_ticket_number, ss_coupon_amt@5 as ss_coupon_amt, ss_net_profit@6 as ss_net_profit, s_city@0 as s_city] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, ss_store_sk@3)], projection=[s_city@1, ss_customer_sk@3, ss_hdemo_sk@4, ss_addr_sk@5, ss_ticket_number@7, ss_coupon_amt@8, ss_net_profit@9] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_customer_sk@3, ss_hdemo_sk@4, ss_addr_sk@5, ss_store_sk@6, ss_ticket_number@7, ss_coupon_amt@8, ss_net_profit@9] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_coupon_amt, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: hd_dep_count@1 = 6 OR hd_vehicle_count@2 > 2, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@1 = 6 OR hd_vehicle_count@2 > 2, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 6 AND 6 <= hd_dep_count_max@1 OR hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_max@4 > 2, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_city@1 as s_city, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_number_employees@1 >= 200 AND s_number_employees@1 <= 295, projection=[s_store_sk@0, s_city@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_number_employees, s_city], file_type=parquet, predicate=s_number_employees@1 >= 200 AND s_number_employees@1 <= 295, pruning_predicate=s_number_employees_null_count@1 != row_count@2 AND s_number_employees_max@0 >= 200 AND s_number_employees_null_count@1 != row_count@2 AND s_number_employees_min@3 <= 295, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_dow@2 = 1 AND (d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001), projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_dow], file_type=parquet, predicate=d_dow@2 = 1 AND (d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001), pruning_predicate=d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 1 AND 1 <= d_dow_max@1 AND (d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_dow in (1), d_year in (1999, 2000, 2001)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_80() -> Result<()> { + let display = test_tpcds_query(80).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [channel@0 ASC, id@1 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, id@1 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[channel@0 as channel, id@1 as id, sum(x.sales)@3 as sales, sum(x.returns_)@4 as returns_, sum(x.profit)@5 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, id@1 as id, __grouping_id@2 as __grouping_id], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([channel@0, id@1, __grouping_id@2], 3), input_partitions=9 + │ AggregateExec: mode=Partial, gby=[(NULL as channel, NULL as id), (channel@0 as channel, NULL as id), (channel@0 as channel, id@1 as id)], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] + │ UnionExec + │ ProjectionExec: expr=[store channel as channel, concat(store, store_id@0) as id, sales@1 as sales, returns_@2 as returns_, profit@3 as profit] + │ ProjectionExec: expr=[s_store_id@0 as store_id, sum(store_sales.ss_ext_sales_price)@1 as sales, sum(coalesce(store_returns.sr_return_amt,Int64(0)))@2 as returns_, sum(store_sales.ss_net_profit - coalesce(store_returns.sr_net_loss,Int64(0)))@3 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_id@0 as s_store_id], aggr=[sum(store_sales.ss_ext_sales_price), sum(coalesce(store_returns.sr_return_amt,Int64(0))), sum(store_sales.ss_net_profit - coalesce(store_returns.sr_net_loss,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([s_store_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[s_store_id@4 as s_store_id], aggr=[sum(store_sales.ss_ext_sales_price), sum(coalesce(store_returns.sr_return_amt,Int64(0))), sum(store_sales.ss_net_profit - coalesce(store_returns.sr_net_loss,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(promotion.p_promo_sk AS Float64)@1, ss_promo_sk@0)], projection=[ss_ext_sales_price@3, ss_net_profit@4, sr_return_amt@5, sr_net_loss@6, s_store_id@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[ss_promo_sk@2, ss_ext_sales_price@3, ss_net_profit@4, sr_return_amt@5, sr_net_loss@6, s_store_id@7] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_promo_sk@2 as ss_promo_sk, ss_ext_sales_price@3 as ss_ext_sales_price, ss_net_profit@4 as ss_net_profit, sr_return_amt@5 as sr_return_amt, sr_net_loss@6 as sr_net_loss, s_store_id@0 as s_store_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, ss_store_sk@1)], projection=[s_store_id@1, ss_item_sk@3, ss_promo_sk@5, ss_ext_sales_price@6, ss_net_profit@7, sr_return_amt@8, sr_net_loss@9] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_promo_sk@5, ss_ext_sales_price@6, ss_net_profit@7, sr_return_amt@8, sr_net_loss@9] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@2 as ss_sold_date_sk, ss_item_sk@3 as ss_item_sk, ss_store_sk@4 as ss_store_sk, ss_promo_sk@5 as ss_promo_sk, ss_ext_sales_price@6 as ss_ext_sales_price, ss_net_profit@7 as ss_net_profit, sr_return_amt@0 as sr_return_amt, sr_net_loss@1 as sr_net_loss] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(sr_item_sk@0, ss_item_sk@1), (sr_ticket_number@1, ss_ticket_number@4)], projection=[sr_return_amt@2, sr_net_loss@3, ss_sold_date_sk@4, ss_item_sk@5, ss_store_sk@6, ss_promo_sk@7, ss_ext_sales_price@9, ss_net_profit@10] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[catalog channel as channel, concat(catalog_page, catalog_page_id@0) as id, sales@1 as sales, returns_@2 as returns_, profit@3 as profit] + │ ProjectionExec: expr=[cp_catalog_page_id@0 as catalog_page_id, sum(catalog_sales.cs_ext_sales_price)@1 as sales, sum(coalesce(catalog_returns.cr_return_amount,Int64(0)))@2 as returns_, sum(catalog_sales.cs_net_profit - coalesce(catalog_returns.cr_net_loss,Int64(0)))@3 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[cp_catalog_page_id@0 as cp_catalog_page_id], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(coalesce(catalog_returns.cr_return_amount,Int64(0))), sum(catalog_sales.cs_net_profit - coalesce(catalog_returns.cr_net_loss,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cp_catalog_page_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cp_catalog_page_id@4 as cp_catalog_page_id], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(coalesce(catalog_returns.cr_return_amount,Int64(0))), sum(catalog_sales.cs_net_profit - coalesce(catalog_returns.cr_net_loss,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(promotion.p_promo_sk AS Float64)@1, cs_promo_sk@0)], projection=[cs_ext_sales_price@3, cs_net_profit@4, cr_return_amount@5, cr_net_loss@6, cp_catalog_page_id@7] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@0)], projection=[cs_promo_sk@2, cs_ext_sales_price@3, cs_net_profit@4, cr_return_amount@5, cr_net_loss@6, cp_catalog_page_id@7] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cs_catalog_page_sk@0, CAST(catalog_page.cp_catalog_page_sk AS Float64)@2)], projection=[cs_item_sk@1, cs_promo_sk@2, cs_ext_sales_price@3, cs_net_profit@4, cr_return_amount@5, cr_net_loss@6, cp_catalog_page_id@8] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_catalog_page_sk@0], 3), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_catalog_page_sk@3, cs_item_sk@4, cs_promo_sk@5, cs_ext_sales_price@6, cs_net_profit@7, cr_return_amount@8, cr_net_loss@9] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@2 as cs_sold_date_sk, cs_catalog_page_sk@3 as cs_catalog_page_sk, cs_item_sk@4 as cs_item_sk, cs_promo_sk@5 as cs_promo_sk, cs_ext_sales_price@6 as cs_ext_sales_price, cs_net_profit@7 as cs_net_profit, cr_return_amount@0 as cr_return_amount, cr_net_loss@1 as cr_net_loss] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(cr_item_sk@0, cs_item_sk@2), (cr_order_number@1, cs_order_number@4)], projection=[cr_return_amount@2, cr_net_loss@3, cs_sold_date_sk@4, cs_catalog_page_sk@5, cs_item_sk@6, cs_promo_sk@7, cs_ext_sales_price@9, cs_net_profit@10] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[web channel as channel, concat(web_site, web_site_id@0) as id, sales@1 as sales, returns_@2 as returns_, profit@3 as profit] + │ ProjectionExec: expr=[web_site_id@0 as web_site_id, sum(web_sales.ws_ext_sales_price)@1 as sales, sum(coalesce(web_returns.wr_return_amt,Int64(0)))@2 as returns_, sum(web_sales.ws_net_profit - coalesce(web_returns.wr_net_loss,Int64(0)))@3 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[web_site_id@0 as web_site_id], aggr=[sum(web_sales.ws_ext_sales_price), sum(coalesce(web_returns.wr_return_amt,Int64(0))), sum(web_sales.ws_net_profit - coalesce(web_returns.wr_net_loss,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([web_site_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[web_site_id@4 as web_site_id], aggr=[sum(web_sales.ws_ext_sales_price), sum(coalesce(web_returns.wr_return_amt,Int64(0))), sum(web_sales.ws_net_profit - coalesce(web_returns.wr_net_loss,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_promo_sk@0, CAST(promotion.p_promo_sk AS Float64)@1)], projection=[ws_ext_sales_price@1, ws_net_profit@2, wr_return_amt@3, wr_net_loss@4, web_site_id@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@0, i_item_sk@0)], projection=[ws_promo_sk@1, ws_ext_sales_price@2, ws_net_profit@3, wr_return_amt@4, wr_net_loss@5, web_site_id@6] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ws_item_sk@1 as ws_item_sk, ws_promo_sk@2 as ws_promo_sk, ws_ext_sales_price@3 as ws_ext_sales_price, ws_net_profit@4 as ws_net_profit, wr_return_amt@5 as wr_return_amt, wr_net_loss@6 as wr_net_loss, web_site_id@0 as web_site_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_site.web_site_sk AS Float64)@2, ws_web_site_sk@1)], projection=[web_site_id@1, ws_item_sk@3, ws_promo_sk@5, ws_ext_sales_price@6, ws_net_profit@7, wr_return_amt@8, wr_net_loss@9] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_web_site_sk@4, ws_promo_sk@5, ws_ext_sales_price@6, ws_net_profit@7, wr_return_amt@8, wr_net_loss@9] + │ CoalescePartitionsExec + │ [Stage 14] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@2 as ws_sold_date_sk, ws_item_sk@3 as ws_item_sk, ws_web_site_sk@4 as ws_web_site_sk, ws_promo_sk@5 as ws_promo_sk, ws_ext_sales_price@6 as ws_ext_sales_price, ws_net_profit@7 as ws_net_profit, wr_return_amt@0 as wr_return_amt, wr_net_loss@1 as wr_net_loss] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(wr_item_sk@0, ws_item_sk@1), (wr_order_number@1, ws_order_number@4)], projection=[wr_return_amt@2, wr_net_loss@3, ws_sold_date_sk@4, ws_item_sk@5, ws_web_site_sk@6, ws_promo_sk@7, ws_ext_sales_price@9, ws_net_profit@10] + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_current_price@1 > Some(5000),4,2, projection=[i_item_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@1 > Some(5000),4,2 AND DynamicFilter [ empty ], pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),4,2, required_guarantees=[] + │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: p_channel_tv@1 = N, projection=[p_promo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, p_channel_tv], file_type=parquet, predicate=p_channel_tv@1 = N, pruning_predicate=p_channel_tv_null_count@2 != row_count@3 AND p_channel_tv_min@0 <= N AND N <= p_channel_tv_max@1, required_guarantees=[p_channel_tv in (N)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: p_channel_tv@1 = N, projection=[p_promo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, p_channel_tv], file_type=parquet, predicate=p_channel_tv@1 = N, pruning_predicate=p_channel_tv_null_count@2 != row_count@3 AND p_channel_tv_min@0 <= N AND N <= p_channel_tv_max@1, required_guarantees=[p_channel_tv in (N)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_current_price@1 > Some(5000),4,2, projection=[i_item_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@1 > Some(5000),4,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),4,2, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_id@1 as s_store_id, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number, sr_return_amt, sr_net_loss], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@1, ss_ticket_number@4], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_ext_sales_price, ss_net_profit], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: p_channel_tv@1 = N, projection=[p_promo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, p_channel_tv], file_type=parquet, predicate=p_channel_tv@1 = N, pruning_predicate=p_channel_tv_null_count@2 != row_count@3 AND p_channel_tv_min@0 <= N AND N <= p_channel_tv_max@1, required_guarantees=[p_channel_tv in (N)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_current_price@1 > Some(5000),4,2, projection=[i_item_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@1 > Some(5000),4,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),4,2, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cr_item_sk@0, cr_order_number@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_return_amount, cr_net_loss], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@2, cs_order_number@4], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_catalog_page_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_ext_sales_price, cs_net_profit], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(catalog_page.cp_catalog_page_sk AS Float64)@2], 3), input_partitions=3 + │ ProjectionExec: expr=[cp_catalog_page_sk@0 as cp_catalog_page_sk, cp_catalog_page_id@1 as cp_catalog_page_id, CAST(cp_catalog_page_sk@0 AS Float64) as CAST(catalog_page.cp_catalog_page_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-3.parquet]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[web_site_sk@0 as web_site_sk, web_site_id@1 as web_site_id, CAST(web_site_sk@0 AS Float64) as CAST(web_site.web_site_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-3.parquet]]}, projection=[web_site_sk, web_site_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([wr_item_sk@0, wr_order_number@1], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_item_sk@1, ws_order_number@4], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_site_sk, ws_promo_sk, ws_order_number, ws_ext_sales_price, ws_net_profit], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_81() -> Result<()> { + let display = test_tpcds_query(81).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c_customer_id@0 ASC NULLS LAST, c_salutation@1 ASC NULLS LAST, c_first_name@2 ASC NULLS LAST, c_last_name@3 ASC NULLS LAST, ca_street_number@4 ASC NULLS LAST, ca_street_name@5 ASC NULLS LAST, ca_street_type@6 ASC NULLS LAST, ca_suite_number@7 ASC NULLS LAST, ca_city@8 ASC NULLS LAST, ca_county@9 ASC NULLS LAST, ca_state@10 ASC NULLS LAST, ca_zip@11 ASC NULLS LAST, ca_country@12 ASC NULLS LAST, ca_gmt_offset@13 ASC NULLS LAST, ca_location_type@14 ASC NULLS LAST, ctr_total_return@15 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[c_customer_id@0 ASC NULLS LAST, c_salutation@1 ASC NULLS LAST, c_first_name@2 ASC NULLS LAST, c_last_name@3 ASC NULLS LAST, ca_street_number@4 ASC NULLS LAST, ca_street_name@5 ASC NULLS LAST, ca_street_type@6 ASC NULLS LAST, ca_suite_number@7 ASC NULLS LAST, ca_city@8 ASC NULLS LAST, ca_county@9 ASC NULLS LAST, ca_zip@11 ASC NULLS LAST, ca_country@12 ASC NULLS LAST, ca_gmt_offset@13 ASC NULLS LAST, ca_location_type@14 ASC NULLS LAST, ctr_total_return@15 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[c_customer_id@12 as c_customer_id, c_salutation@13 as c_salutation, c_first_name@14 as c_first_name, c_last_name@15 as c_last_name, ca_street_number@1 as ca_street_number, ca_street_name@2 as ca_street_name, ca_street_type@3 as ca_street_type, ca_suite_number@4 as ca_suite_number, ca_city@5 as ca_city, ca_county@6 as ca_county, ca_state@7 as ca_state, ca_zip@8 as ca_zip, ca_country@9 as ca_country, ca_gmt_offset@10 as ca_gmt_offset, ca_location_type@11 as ca_location_type, ctr_total_return@0 as ctr_total_return] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ctr_state@0, ctr_state@1)], filter=CAST(ctr_total_return@0 AS Decimal128(30, 15)) > avg(ctr2.ctr_total_return) * Float64(1.2)@1, projection=[ctr_total_return@1, ca_street_number@2, ca_street_name@3, ca_street_type@4, ca_suite_number@5, ca_city@6, ca_county@7, ca_state@8, ca_zip@9, ca_country@10, ca_gmt_offset@11, ca_location_type@12, c_customer_id@13, c_salutation@14, c_first_name@15, c_last_name@16] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ctr_state@11 as ctr_state, ctr_total_return@12 as ctr_total_return, ca_street_number@0 as ca_street_number, ca_street_name@1 as ca_street_name, ca_street_type@2 as ca_street_type, ca_suite_number@3 as ca_suite_number, ca_city@4 as ca_city, ca_county@5 as ca_county, ca_state@6 as ca_state, ca_zip@7 as ca_zip, ca_country@8 as ca_country, ca_gmt_offset@9 as ca_gmt_offset, ca_location_type@10 as ca_location_type, c_customer_id@13 as c_customer_id, c_salutation@14 as c_salutation, c_first_name@15 as c_first_name, c_last_name@16 as c_last_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@3)], projection=[ca_street_number@1, ca_street_name@2, ca_street_type@3, ca_suite_number@4, ca_city@5, ca_county@6, ca_state@7, ca_zip@8, ca_country@9, ca_gmt_offset@10, ca_location_type@11, ctr_state@12, ctr_total_return@13, c_customer_id@14, c_salutation@16, c_first_name@17, c_last_name@18] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ctr_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@6)], projection=[ctr_state@1, ctr_total_return@2, c_customer_id@4, c_current_addr_sk@5, c_salutation@6, c_first_name@7, c_last_name@8] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[cr_returning_customer_sk@0 as ctr_customer_sk, ca_state@1 as ctr_state, sum(catalog_returns.cr_return_amt_inc_tax)@2 as ctr_total_return] + │ AggregateExec: mode=FinalPartitioned, gby=[cr_returning_customer_sk@0 as cr_returning_customer_sk, ca_state@1 as ca_state], aggr=[sum(catalog_returns.cr_return_amt_inc_tax)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cr_returning_customer_sk@0, ca_state@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cr_returning_customer_sk@0 as cr_returning_customer_sk, ca_state@2 as ca_state], aggr=[sum(catalog_returns.cr_return_amt_inc_tax)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cr_returning_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@2)], projection=[cr_returning_customer_sk@0, cr_return_amt_inc_tax@2, ca_state@4] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cr_returned_date_sk@0)], projection=[cr_returning_customer_sk@2, cr_returning_addr_sk@3, cr_return_amt_inc_tax@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_returning_addr_sk, cr_return_amt_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_state@1 as ca_state, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state], file_type=parquet + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_current_addr_sk@2 as c_current_addr_sk, c_salutation@3 as c_salutation, c_first_name@4 as c_first_name, c_last_name@5 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_current_addr_sk, c_salutation, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[CAST(CAST(avg(ctr2.ctr_total_return)@1 AS Float64) * 1.2 AS Decimal128(30, 15)) as avg(ctr2.ctr_total_return) * Float64(1.2), ctr_state@0 as ctr_state] + │ AggregateExec: mode=FinalPartitioned, gby=[ctr_state@0 as ctr_state], aggr=[avg(ctr2.ctr_total_return)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ctr_state@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ctr_state@0 as ctr_state], aggr=[avg(ctr2.ctr_total_return)] + │ ProjectionExec: expr=[ca_state@1 as ctr_state, sum(catalog_returns.cr_return_amt_inc_tax)@2 as ctr_total_return] + │ AggregateExec: mode=FinalPartitioned, gby=[cr_returning_customer_sk@0 as cr_returning_customer_sk, ca_state@1 as ca_state], aggr=[sum(catalog_returns.cr_return_amt_inc_tax)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cr_returning_customer_sk@0, ca_state@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cr_returning_customer_sk@0 as cr_returning_customer_sk, ca_state@2 as ca_state], aggr=[sum(catalog_returns.cr_return_amt_inc_tax)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cr_returning_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@2)], projection=[cr_returning_customer_sk@0, cr_return_amt_inc_tax@2, ca_state@4] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cr_returned_date_sk@0)], projection=[cr_returning_customer_sk@2, cr_returning_addr_sk@3, cr_return_amt_inc_tax@4] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_returning_addr_sk, cr_return_amt_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_state@1 as ca_state, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ca_state@7 = GA + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_street_type, ca_suite_number, ca_city, ca_county, ca_state, ca_zip, ca_country, ca_gmt_offset, ca_location_type], file_type=parquet, predicate=ca_state@7 = GA, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1, required_guarantees=[ca_state in (GA)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_82() -> Result<()> { + let display = test_tpcds_query(82).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, i_current_price@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[i_item_id@1, i_item_desc@2, i_current_price@3] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(inv_date_sk@4, d_date_sk@0)], projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[i_item_sk@1 as i_item_sk, i_item_id@2 as i_item_id, i_item_desc@3 as i_item_desc, i_current_price@4 as i_current_price, inv_date_sk@0 as inv_date_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(inv_item_sk@1, i_item_sk@0)], projection=[inv_date_sk@0, i_item_sk@2, i_item_id@3, i_item_desc@4, i_current_price@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_current_price@3 >= Some(6200),4,2 AND i_current_price@3 <= Some(9200),4,2 AND i_manufact_id@4 IN (SET) ([129, 270, 821, 423]), projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@3 >= Some(6200),4,2 AND i_current_price@3 <= Some(9200),4,2 AND i_manufact_id@4 IN (SET) ([129, 270, 821, 423]) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(6200),4,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(9200),4,2 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 129 AND 129 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 270 AND 270 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 821 AND 821 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 423 AND 423 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (129, 270, 423, 821)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-05-25 AND d_date@1 <= 2000-07-24, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-05-25 AND d_date@1 <= 2000-07-24 AND DynamicFilter [ empty ], pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-05-25 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-07-24, required_guarantees=[] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: inv_quantity_on_hand@2 >= 100 AND inv_quantity_on_hand@2 <= 500, projection=[inv_date_sk@0, inv_item_sk@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], ...]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@2 >= 100 AND inv_quantity_on_hand@2 <= 500, pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_83() -> Result<()> { + let display = test_tpcds_query(83).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [item_id@0 ASC, sr_item_qty@1 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[item_id@0 ASC, sr_item_qty@1 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[item_id@1 as item_id, sr_item_qty@2 as sr_item_qty, sr_item_qty@2 / __common_expr_7@0 / 3 * 100 as sr_dev, cr_item_qty@3 as cr_item_qty, cr_item_qty@3 / __common_expr_7@0 / 3 * 100 as cr_dev, wr_item_qty@4 as wr_item_qty, wr_item_qty@4 / __common_expr_7@0 / 3 * 100 as wr_dev, __common_expr_7@0 / 3 as average] + │ ProjectionExec: expr=[sr_item_qty@2 + cr_item_qty@3 + wr_item_qty@0 as __common_expr_7, item_id@1 as item_id, sr_item_qty@2 as sr_item_qty, cr_item_qty@3 as cr_item_qty, wr_item_qty@0 as wr_item_qty] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(item_id@0, item_id@0)], projection=[wr_item_qty@1, item_id@2, sr_item_qty@3, cr_item_qty@4] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[i_item_id@0 as item_id, sum(web_returns.wr_return_quantity)@1 as wr_item_qty] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(web_returns.wr_return_quantity)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(web_returns.wr_return_quantity)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(d_date@2, d_date@0)], projection=[wr_return_quantity@0, i_item_id@1] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(wr_returned_date_sk@0, CAST(date_dim.d_date_sk AS Float64)@2)], projection=[wr_return_quantity@1, i_item_id@2, d_date@4] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@0] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(item_id@0, item_id@0)], projection=[item_id@0, sr_item_qty@1, cr_item_qty@3] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[i_item_id@0 as item_id, sum(store_returns.sr_return_quantity)@1 as sr_item_qty] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(store_returns.sr_return_quantity)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(store_returns.sr_return_quantity)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date@0, d_date@2)], projection=[sr_return_quantity@0, i_item_id@1] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@0] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet + │ ProjectionExec: expr=[sr_return_quantity@1 as sr_return_quantity, i_item_id@2 as i_item_id, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, sr_returned_date_sk@0)], projection=[d_date@1, sr_return_quantity@4, i_item_id@5] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[i_item_id@0 as item_id, sum(catalog_returns.cr_return_quantity)@1 as cr_item_qty] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(catalog_returns.cr_return_quantity)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(catalog_returns.cr_return_quantity)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date@0, d_date@2)], projection=[cr_return_quantity@0, i_item_id@1] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@0] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet + │ ProjectionExec: expr=[cr_return_quantity@1 as cr_return_quantity, i_item_id@2 as i_item_id, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(d_date_sk@0, cr_returned_date_sk@0)], projection=[d_date@1, cr_return_quantity@3, i_item_id@4] + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ ProjectionExec: expr=[wr_returned_date_sk@1 as wr_returned_date_sk, wr_return_quantity@2 as wr_return_quantity, i_item_id@0 as i_item_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, wr_item_sk@1)], projection=[i_item_id@1, wr_returned_date_sk@2, wr_return_quantity@4] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([wr_item_sk@1], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@0 = 2000-06-30 OR d_date@0 = 2000-09-27 OR d_date@0 = 2000-11-17, projection=[d_week_seq@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@0 = 2000-06-30 OR d_date@0 = 2000-09-27 OR d_date@0 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@0 = 2000-06-30 OR d_date@0 = 2000-09-27 OR d_date@0 = 2000-11-17, projection=[d_week_seq@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@0 = 2000-06-30 OR d_date@0 = 2000-09-27 OR d_date@0 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@2], 3), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_returned_date_sk@0], 3), input_partitions=3 + │ ProjectionExec: expr=[sr_returned_date_sk@1 as sr_returned_date_sk, sr_return_quantity@2 as sr_return_quantity, i_item_id@0 as i_item_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, sr_item_sk@1)], projection=[i_item_id@1, sr_returned_date_sk@2, sr_return_quantity@4] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_item_sk@1], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_returned_date_sk, sr_item_sk, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@0 = 2000-06-30 OR d_date@0 = 2000-09-27 OR d_date@0 = 2000-11-17, projection=[d_week_seq@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@0 = 2000-06-30 OR d_date@0 = 2000-09-27 OR d_date@0 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([d_date_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cr_returned_date_sk@0], 3), input_partitions=3 + │ ProjectionExec: expr=[cr_returned_date_sk@1 as cr_returned_date_sk, cr_return_quantity@2 as cr_return_quantity, i_item_id@0 as i_item_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cr_item_sk@1)], projection=[i_item_id@1, cr_returned_date_sk@2, cr_return_quantity@4] + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cr_item_sk@1], 6), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_returned_date_sk, cr_item_sk, cr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_84() -> Result<()> { + let display = test_tpcds_query(84).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[customer_id@0 as customer_id, customername@1 as customername] + │ SortPreservingMergeExec: [c_customer_id@2 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[customer_id@0 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, concat(concat(coalesce(c_last_name@2, ), , ), coalesce(c_first_name@1, )) as customername, c_customer_id@0 as c_customer_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@4, sr_cdemo_sk@0)], projection=[c_customer_id@0, c_first_name@1, c_last_name@2] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, cd_demo_sk@3 as cd_demo_sk, CAST(cd_demo_sk@3 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ib_income_band_sk@0, hd_income_band_sk@4)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, cd_demo_sk@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, cd_demo_sk@4 as cd_demo_sk, hd_income_band_sk@0 as hd_income_band_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@2, c_current_hdemo_sk@1)], projection=[hd_income_band_sk@1, c_customer_id@3, c_first_name@5, c_last_name@6, cd_demo_sk@7] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@1, CAST(customer_demographics.cd_demo_sk AS Float64)@1)], projection=[c_customer_id@0, c_current_hdemo_sk@2, c_first_name@3, c_last_name@4, cd_demo_sk@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@3)], projection=[c_customer_id@1, c_current_cdemo_sk@2, c_current_hdemo_sk@3, c_first_name@5, c_last_name@6] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_id, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk], file_type=parquet + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_cdemo_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ib_lower_bound@1 >= 38128 AND ib_upper_bound@2 <= 88128, projection=[ib_income_band_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet]]}, projection=[ib_income_band_sk, ib_lower_bound, ib_upper_bound], file_type=parquet, predicate=ib_lower_bound@1 >= 38128 AND ib_upper_bound@2 <= 88128, pruning_predicate=ib_lower_bound_null_count@1 != row_count@2 AND ib_lower_bound_max@0 >= 38128 AND ib_upper_bound_null_count@4 != row_count@2 AND ib_upper_bound_min@3 <= 88128, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, hd_income_band_sk@1 as hd_income_band_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ca_city@1 = Edgewood, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_city], file_type=parquet, predicate=ca_city@1 = Edgewood, pruning_predicate=ca_city_null_count@2 != row_count@3 AND ca_city_min@0 <= Edgewood AND Edgewood <= ca_city_max@1, required_guarantees=[ca_city in (Edgewood)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_85() -> Result<()> { + let display = test_tpcds_query(85).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[substr(reason.r_reason_desc,Int64(1),Int64(20))@0 as substr(reason.r_reason_desc,Int64(1),Int64(20)), avg1@1 as avg1, avg2@2 as avg2, avg(web_returns.wr_fee)@3 as avg(web_returns.wr_fee)] + │ SortPreservingMergeExec: [substr(reason.r_reason_desc,Int64(1),Int64(20))@0 ASC NULLS LAST, avg(web_sales.ws_quantity)@4 ASC NULLS LAST, avg(web_returns.wr_refunded_cash)@5 ASC NULLS LAST, avg(web_returns.wr_fee)@3 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[substr(reason.r_reason_desc,Int64(1),Int64(20))@0 ASC NULLS LAST, avg1@1 ASC NULLS LAST, avg2@2 ASC NULLS LAST, avg(web_returns.wr_fee)@3 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[substr(r_reason_desc@0, 1, 20) as substr(reason.r_reason_desc,Int64(1),Int64(20)), avg(web_sales.ws_quantity)@1 as avg1, avg(web_returns.wr_refunded_cash)@2 as avg2, avg(web_returns.wr_fee)@3 as avg(web_returns.wr_fee), avg(web_sales.ws_quantity)@1 as avg(web_sales.ws_quantity), avg(web_returns.wr_refunded_cash)@2 as avg(web_returns.wr_refunded_cash)] + │ AggregateExec: mode=FinalPartitioned, gby=[r_reason_desc@0 as r_reason_desc], aggr=[avg(web_sales.ws_quantity), avg(web_returns.wr_refunded_cash), avg(web_returns.wr_fee)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([r_reason_desc@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[r_reason_desc@3 as r_reason_desc], aggr=[avg(web_sales.ws_quantity), avg(web_returns.wr_refunded_cash), avg(web_returns.wr_fee)] + │ ProjectionExec: expr=[ws_quantity@1 as ws_quantity, wr_fee@2 as wr_fee, wr_refunded_cash@3 as wr_refunded_cash, r_reason_desc@0 as r_reason_desc] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(reason.r_reason_sk AS Float64)@2, wr_reason_sk@1)], projection=[r_reason_desc@1, ws_quantity@3, wr_fee@5, wr_refunded_cash@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_sold_date_sk@0, CAST(date_dim.d_date_sk AS Float64)@1)], projection=[ws_quantity@1, wr_reason_sk@2, wr_fee@3, wr_refunded_cash@4] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(wr_refunded_addr_sk@3, CAST(customer_address.ca_address_sk AS Float64)@2)], filter=(ca_state@1 = IN OR ca_state@1 = OH OR ca_state@1 = NJ) AND ws_net_profit@0 >= Some(10000),7,2 AND ws_net_profit@0 <= Some(20000),7,2 OR (ca_state@1 = WI OR ca_state@1 = CT OR ca_state@1 = KY) AND ws_net_profit@0 >= Some(15000),7,2 AND ws_net_profit@0 <= Some(30000),7,2 OR (ca_state@1 = LA OR ca_state@1 = IA OR ca_state@1 = AR) AND ws_net_profit@0 >= Some(5000),7,2 AND ws_net_profit@0 <= Some(25000),7,2, projection=[ws_sold_date_sk@0, ws_quantity@1, wr_reason_sk@4, wr_fee@5, wr_refunded_cash@6] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(wr_returning_cdemo_sk@4, CAST(cd2.cd_demo_sk AS Float64)@3), (cd_marital_status@8, cd_marital_status@1), (cd_education_status@9, cd_education_status@2)], projection=[ws_sold_date_sk@0, ws_quantity@1, ws_net_profit@2, wr_refunded_addr_sk@3, wr_reason_sk@5, wr_fee@6, wr_refunded_cash@7] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(wr_refunded_cdemo_sk@4, CAST(cd1.cd_demo_sk AS Float64)@3)], filter=cd_marital_status@1 = M AND cd_education_status@2 = Advanced Degree AND ws_sales_price@0 >= Some(10000),5,2 AND ws_sales_price@0 <= Some(15000),5,2 OR cd_marital_status@1 = S AND cd_education_status@2 = College AND ws_sales_price@0 >= Some(5000),5,2 AND ws_sales_price@0 <= Some(10000),5,2 OR cd_marital_status@1 = W AND cd_education_status@2 = 2 yr Degree AND ws_sales_price@0 >= Some(15000),5,2 AND ws_sales_price@0 <= Some(20000),5,2, projection=[ws_sold_date_sk@0, ws_quantity@1, ws_net_profit@3, wr_refunded_addr_sk@5, wr_returning_cdemo_sk@6, wr_reason_sk@7, wr_fee@8, wr_refunded_cash@9, cd_marital_status@11, cd_education_status@12] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_page.wp_web_page_sk AS Float64)@1, ws_web_page_sk@1)], projection=[ws_sold_date_sk@2, ws_quantity@4, ws_sales_price@5, ws_net_profit@6, wr_refunded_cdemo_sk@7, wr_refunded_addr_sk@8, wr_returning_cdemo_sk@9, wr_reason_sk@10, wr_fee@11, wr_refunded_cash@12] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@6 as ws_sold_date_sk, ws_web_page_sk@7 as ws_web_page_sk, ws_quantity@8 as ws_quantity, ws_sales_price@9 as ws_sales_price, ws_net_profit@10 as ws_net_profit, wr_refunded_cdemo_sk@0 as wr_refunded_cdemo_sk, wr_refunded_addr_sk@1 as wr_refunded_addr_sk, wr_returning_cdemo_sk@2 as wr_returning_cdemo_sk, wr_reason_sk@3 as wr_reason_sk, wr_fee@4 as wr_fee, wr_refunded_cash@5 as wr_refunded_cash] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(wr_item_sk@0, ws_item_sk@1), (wr_order_number@5, ws_order_number@3)], projection=[wr_refunded_cdemo_sk@1, wr_refunded_addr_sk@2, wr_returning_cdemo_sk@3, wr_reason_sk@4, wr_fee@6, wr_refunded_cash@7, ws_sold_date_sk@8, ws_web_page_sk@10, ws_quantity@12, ws_sales_price@13, ws_net_profit@14] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, CAST(cd_demo_sk@0 AS Float64) as CAST(cd1.cd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cd_marital_status@1 = M AND cd_education_status@2 = Advanced Degree OR cd_marital_status@1 = S AND cd_education_status@2 = College OR cd_marital_status@1 = W AND cd_education_status@2 = 2 yr Degree + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_marital_status@1 = M AND cd_education_status@2 = Advanced Degree OR cd_marital_status@1 = S AND cd_education_status@2 = College OR cd_marital_status@1 = W AND cd_education_status@2 = 2 yr Degree, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, Advanced Degree, College), cd_marital_status in (M, S, W)] + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, CAST(cd_demo_sk@0 AS Float64) as CAST(cd2.cd_demo_sk AS Float64)] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_state@1 as ca_state, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: (ca_state@1 = IN OR ca_state@1 = OH OR ca_state@1 = NJ OR ca_state@1 = WI OR ca_state@1 = CT OR ca_state@1 = KY OR ca_state@1 = LA OR ca_state@1 = IA OR ca_state@1 = AR) AND ca_state@1 IN ([IN, OH, NJ, WI, CT, KY, LA, IA, AR]) AND ca_country@2 = United States, projection=[ca_address_sk@0, ca_state@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=(ca_state@1 = IN OR ca_state@1 = OH OR ca_state@1 = NJ OR ca_state@1 = WI OR ca_state@1 = CT OR ca_state@1 = KY OR ca_state@1 = LA OR ca_state@1 = IA OR ca_state@1 = AR) AND ca_state@1 IN ([IN, OH, NJ, WI, CT, KY, LA, IA, AR]) AND ca_country@2 = United States, pruning_predicate=(ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IN AND IN <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OH AND OH <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NJ AND NJ <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= WI AND WI <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= CT AND CT <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= KY AND KY <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= LA AND LA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IA AND IA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= AR AND AR <= ca_state_max@1) AND (ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IN AND IN <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OH AND OH <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NJ AND NJ <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= WI AND WI <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= CT AND CT <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= KY AND KY <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= LA AND LA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IA AND IA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= AR AND AR <= ca_state_max@1) AND ca_country_null_count@6 != row_count@3 AND ca_country_min@4 <= United States AND United States <= ca_country_max@5, required_guarantees=[ca_country in (United States), ca_state in (AR, CT, IA, IN, KY, LA, NJ, OH, WI)] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[r_reason_sk@0 as r_reason_sk, r_reason_desc@1 as r_reason_desc, CAST(r_reason_sk@0 AS Float64) as CAST(reason.r_reason_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/reason/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-3.parquet]]}, projection=[r_reason_sk, r_reason_desc], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[wp_web_page_sk@0 as wp_web_page_sk, CAST(wp_web_page_sk@0 AS Float64) as CAST(web_page.wp_web_page_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-3.parquet]]}, projection=[wp_web_page_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([wr_item_sk@0, wr_order_number@5], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_refunded_cdemo_sk, wr_refunded_addr_sk, wr_returning_cdemo_sk, wr_reason_sk, wr_order_number, wr_fee, wr_refunded_cash], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_item_sk@1, ws_order_number@3], 3), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: (ws_net_profit@6 >= Some(10000),7,2 AND ws_net_profit@6 <= Some(20000),7,2 OR ws_net_profit@6 >= Some(15000),7,2 AND ws_net_profit@6 <= Some(30000),7,2 OR ws_net_profit@6 >= Some(5000),7,2 AND ws_net_profit@6 <= Some(25000),7,2) AND (ws_sales_price@5 >= Some(10000),5,2 AND ws_sales_price@5 <= Some(15000),5,2 OR ws_sales_price@5 >= Some(5000),5,2 AND ws_sales_price@5 <= Some(10000),5,2 OR ws_sales_price@5 >= Some(15000),5,2 AND ws_sales_price@5 <= Some(20000),5,2) + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_page_sk, ws_order_number, ws_quantity, ws_sales_price, ws_net_profit], file_type=parquet, predicate=(ws_net_profit@6 >= Some(10000),7,2 AND ws_net_profit@6 <= Some(20000),7,2 OR ws_net_profit@6 >= Some(15000),7,2 AND ws_net_profit@6 <= Some(30000),7,2 OR ws_net_profit@6 >= Some(5000),7,2 AND ws_net_profit@6 <= Some(25000),7,2) AND (ws_sales_price@5 >= Some(10000),5,2 AND ws_sales_price@5 <= Some(15000),5,2 OR ws_sales_price@5 >= Some(5000),5,2 AND ws_sales_price@5 <= Some(10000),5,2 OR ws_sales_price@5 >= Some(15000),5,2 AND ws_sales_price@5 <= Some(20000),5,2) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=(ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(10000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(20000),7,2 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(15000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(30000),7,2 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(5000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(25000),7,2) AND (ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(10000),5,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(15000),5,2 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(5000),5,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(10000),5,2 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(15000),5,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(20000),5,2), required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + #[ignore = "The ordering of the column names in the first nodes is non deterministickI"] + async fn test_tpcds_86() -> Result<()> { + let display = test_tpcds_query(86).await?; + assert_snapshot!(display, @r#""#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_87() -> Result<()> { + let display = test_tpcds_query(87).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(c_last_name@0, c_last_name@0), (c_first_name@1, c_first_name@1), (d_date@2, d_date@2)], NullsEqual: true + │ CoalescePartitionsExec + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, d_date@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ ProjectionExec: expr=[c_last_name@2 as c_last_name, c_first_name@1 as c_first_name, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_bill_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@3)], projection=[d_date@1, c_first_name@3, c_last_name@4] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ws_bill_customer_sk@1 as ws_bill_customer_sk, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@0)], projection=[d_date@1, ws_bill_customer_sk@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ AggregateExec: mode=SinglePartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(c_last_name@0, c_last_name@0), (c_first_name@1, c_first_name@1), (d_date@2, d_date@2)], NullsEqual: true + │ CoalescePartitionsExec + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, d_date@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ ProjectionExec: expr=[c_last_name@2 as c_last_name, c_first_name@1 as c_first_name, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_bill_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@3)], projection=[d_date@1, c_first_name@3, c_last_name@4] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[cs_bill_customer_sk@1 as cs_bill_customer_sk, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@0)], projection=[d_date@1, cs_bill_customer_sk@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, d_date@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ ProjectionExec: expr=[c_last_name@2 as c_last_name, c_first_name@1 as c_first_name, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@3)], projection=[d_date@1, c_first_name@3, c_last_name@4] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_customer_sk@1 as ss_customer_sk, d_date@0 as d_date] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_date@1, ss_customer_sk@4] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_88() -> Result<()> { + let display = test_tpcds_query(88).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[h8_30_to_9@1 as h8_30_to_9, h9_to_9_30@2 as h9_to_9_30, h9_30_to_10@3 as h9_30_to_10, h10_to_10_30@4 as h10_to_10_30, h10_30_to_11@5 as h10_30_to_11, h11_to_11_30@6 as h11_to_11_30, h11_30_to_12@7 as h11_30_to_12, h12_to_12_30@0 as h12_to_12_30] + │ CrossJoinExec + │ ProjectionExec: expr=[count(Int64(1))@0 as h12_to_12_30] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ss_sold_time_sk@0)], projection=[ss_store_sk@3] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_sold_time_sk@2, ss_store_sk@4] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[h8_30_to_9@1 as h8_30_to_9, h9_to_9_30@2 as h9_to_9_30, h9_30_to_10@3 as h9_30_to_10, h10_to_10_30@4 as h10_to_10_30, h10_30_to_11@5 as h10_30_to_11, h11_to_11_30@6 as h11_to_11_30, h11_30_to_12@0 as h11_30_to_12] + │ CrossJoinExec + │ ProjectionExec: expr=[count(Int64(1))@0 as h11_30_to_12] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ss_sold_time_sk@0)], projection=[ss_store_sk@3] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_sold_time_sk@2, ss_store_sk@4] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[h8_30_to_9@1 as h8_30_to_9, h9_to_9_30@2 as h9_to_9_30, h9_30_to_10@3 as h9_30_to_10, h10_to_10_30@4 as h10_to_10_30, h10_30_to_11@5 as h10_30_to_11, h11_to_11_30@0 as h11_to_11_30] + │ CrossJoinExec + │ ProjectionExec: expr=[count(Int64(1))@0 as h11_to_11_30] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ss_sold_time_sk@0)], projection=[ss_store_sk@3] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_sold_time_sk@2, ss_store_sk@4] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[h8_30_to_9@1 as h8_30_to_9, h9_to_9_30@2 as h9_to_9_30, h9_30_to_10@3 as h9_30_to_10, h10_to_10_30@4 as h10_to_10_30, h10_30_to_11@0 as h10_30_to_11] + │ CrossJoinExec + │ ProjectionExec: expr=[count(Int64(1))@0 as h10_30_to_11] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ss_sold_time_sk@0)], projection=[ss_store_sk@3] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_sold_time_sk@2, ss_store_sk@4] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[h8_30_to_9@1 as h8_30_to_9, h9_to_9_30@2 as h9_to_9_30, h9_30_to_10@3 as h9_30_to_10, h10_to_10_30@0 as h10_to_10_30] + │ CrossJoinExec + │ ProjectionExec: expr=[count(Int64(1))@0 as h10_to_10_30] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ss_sold_time_sk@0)], projection=[ss_store_sk@3] + │ CoalescePartitionsExec + │ [Stage 14] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_sold_time_sk@2, ss_store_sk@4] + │ CoalescePartitionsExec + │ [Stage 15] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[h8_30_to_9@1 as h8_30_to_9, h9_to_9_30@2 as h9_to_9_30, h9_30_to_10@0 as h9_30_to_10] + │ CrossJoinExec + │ ProjectionExec: expr=[count(Int64(1))@0 as h9_30_to_10] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)] + │ CoalescePartitionsExec + │ [Stage 16] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ss_sold_time_sk@0)], projection=[ss_store_sk@3] + │ CoalescePartitionsExec + │ [Stage 17] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_sold_time_sk@2, ss_store_sk@4] + │ CoalescePartitionsExec + │ [Stage 18] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ CrossJoinExec + │ ProjectionExec: expr=[count(Int64(1))@0 as h8_30_to_9] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)] + │ CoalescePartitionsExec + │ [Stage 19] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ss_sold_time_sk@0)], projection=[ss_store_sk@3] + │ CoalescePartitionsExec + │ [Stage 20] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_sold_time_sk@2, ss_store_sk@4] + │ CoalescePartitionsExec + │ [Stage 21] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[count(Int64(1))@0 as h9_to_9_30] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)] + │ CoalescePartitionsExec + │ [Stage 22] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ss_sold_time_sk@0)], projection=[ss_store_sk@3] + │ CoalescePartitionsExec + │ [Stage 23] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_sold_time_sk@2, ss_store_sk@4] + │ CoalescePartitionsExec + │ [Stage 24] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_store_name@1 = ese, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name], file_type=parquet, predicate=s_store_name@1 = ese, pruning_predicate=s_store_name_null_count@2 != row_count@3 AND s_store_name_min@0 <= ese AND ese <= s_store_name_max@1, required_guarantees=[s_store_name in (ese)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: t_hour@1 = 12 AND t_minute@2 < 30, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@1 = 12 AND t_minute@2 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 12 AND 12 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (12)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 4 AND 4 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 6 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 2 AND 2 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 4 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 0 AND 0 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 2, required_guarantees=[hd_dep_count in (0, 2, 4)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_store_name@1 = ese, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name], file_type=parquet, predicate=s_store_name@1 = ese, pruning_predicate=s_store_name_null_count@2 != row_count@3 AND s_store_name_min@0 <= ese AND ese <= s_store_name_max@1, required_guarantees=[s_store_name in (ese)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: t_hour@1 = 11 AND t_minute@2 >= 30, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@1 = 11 AND t_minute@2 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 11 AND 11 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (11)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 4 AND 4 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 6 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 2 AND 2 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 4 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 0 AND 0 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 2, required_guarantees=[hd_dep_count in (0, 2, 4)] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_store_name@1 = ese, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name], file_type=parquet, predicate=s_store_name@1 = ese, pruning_predicate=s_store_name_null_count@2 != row_count@3 AND s_store_name_min@0 <= ese AND ese <= s_store_name_max@1, required_guarantees=[s_store_name in (ese)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: t_hour@1 = 11 AND t_minute@2 < 30, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@1 = 11 AND t_minute@2 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 11 AND 11 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (11)] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 4 AND 4 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 6 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 2 AND 2 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 4 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 0 AND 0 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 2, required_guarantees=[hd_dep_count in (0, 2, 4)] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_store_name@1 = ese, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name], file_type=parquet, predicate=s_store_name@1 = ese, pruning_predicate=s_store_name_null_count@2 != row_count@3 AND s_store_name_min@0 <= ese AND ese <= s_store_name_max@1, required_guarantees=[s_store_name in (ese)] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: t_hour@1 = 10 AND t_minute@2 >= 30, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@1 = 10 AND t_minute@2 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 10 AND 10 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (10)] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 4 AND 4 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 6 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 2 AND 2 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 4 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 0 AND 0 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 2, required_guarantees=[hd_dep_count in (0, 2, 4)] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_store_name@1 = ese, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name], file_type=parquet, predicate=s_store_name@1 = ese, pruning_predicate=s_store_name_null_count@2 != row_count@3 AND s_store_name_min@0 <= ese AND ese <= s_store_name_max@1, required_guarantees=[s_store_name in (ese)] + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: t_hour@1 = 10 AND t_minute@2 < 30, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@1 = 10 AND t_minute@2 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 10 AND 10 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (10)] + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 4 AND 4 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 6 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 2 AND 2 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 4 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 0 AND 0 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 2, required_guarantees=[hd_dep_count in (0, 2, 4)] + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_store_name@1 = ese, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name], file_type=parquet, predicate=s_store_name@1 = ese, pruning_predicate=s_store_name_null_count@2 != row_count@3 AND s_store_name_min@0 <= ese AND ese <= s_store_name_max@1, required_guarantees=[s_store_name in (ese)] + └────────────────────────────────────────────────── + ┌───── Stage 17 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: t_hour@1 = 9 AND t_minute@2 >= 30, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@1 = 9 AND t_minute@2 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 9 AND 9 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (9)] + └────────────────────────────────────────────────── + ┌───── Stage 18 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 4 AND 4 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 6 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 2 AND 2 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 4 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 0 AND 0 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 2, required_guarantees=[hd_dep_count in (0, 2, 4)] + └────────────────────────────────────────────────── + ┌───── Stage 19 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_store_name@1 = ese, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name], file_type=parquet, predicate=s_store_name@1 = ese, pruning_predicate=s_store_name_null_count@2 != row_count@3 AND s_store_name_min@0 <= ese AND ese <= s_store_name_max@1, required_guarantees=[s_store_name in (ese)] + └────────────────────────────────────────────────── + ┌───── Stage 20 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: t_hour@1 = 8 AND t_minute@2 >= 30, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@1 = 8 AND t_minute@2 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 8 AND 8 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (8)] + └────────────────────────────────────────────────── + ┌───── Stage 21 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 4 AND 4 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 6 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 2 AND 2 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 4 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 0 AND 0 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 2, required_guarantees=[hd_dep_count in (0, 2, 4)] + └────────────────────────────────────────────────── + ┌───── Stage 22 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_store_name@1 = ese, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name], file_type=parquet, predicate=s_store_name@1 = ese, pruning_predicate=s_store_name_null_count@2 != row_count@3 AND s_store_name_min@0 <= ese AND ese <= s_store_name_max@1, required_guarantees=[s_store_name in (ese)] + └────────────────────────────────────────────────── + ┌───── Stage 23 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: t_hour@1 = 9 AND t_minute@2 < 30, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@1 = 9 AND t_minute@2 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 9 AND 9 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (9)] + └────────────────────────────────────────────────── + ┌───── Stage 24 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 4 AND 4 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 6 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 2 AND 2 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 4 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 0 AND 0 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 2, required_guarantees=[hd_dep_count in (0, 2, 4)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_89() -> Result<()> { + let display = test_tpcds_query(89).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [sum_sales@6 - avg_monthly_sales@7 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST, i_category@0 ASC NULLS LAST, i_class@1 ASC NULLS LAST, i_brand@2 ASC NULLS LAST, s_company_name@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST, sum_sales@6 ASC NULLS LAST, avg_monthly_sales@7 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[sum_sales@6 - avg_monthly_sales@7 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST, i_category@0 ASC NULLS LAST, i_class@1 ASC NULLS LAST, i_brand@2 ASC NULLS LAST, s_company_name@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST, sum_sales@6 ASC NULLS LAST, avg_monthly_sales@7 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_category@0 as i_category, i_class@1 as i_class, i_brand@2 as i_brand, s_store_name@3 as s_store_name, s_company_name@4 as s_company_name, d_moy@5 as d_moy, sum(store_sales.ss_sales_price)@6 as sum_sales, avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@7 as avg_monthly_sales] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CASE WHEN avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@7 != Some(0),19,6 THEN abs(sum(store_sales.ss_sales_price)@6 - avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@7) / avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@7 END > Some(1000000000),30,10 + │ WindowAggExec: wdw=[avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Decimal128(19, 6), nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@2 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST, s_company_name@4 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@2, s_store_name@3, s_company_name@4], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_class@1 as i_class, i_brand@2 as i_brand, s_store_name@3 as s_store_name, s_company_name@4 as s_company_name, d_moy@5 as d_moy], aggr=[sum(store_sales.ss_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_class@1, i_brand@2, s_store_name@3, s_company_name@4, d_moy@5], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@2 as i_category, i_class@1 as i_class, i_brand@0 as i_brand, s_store_name@5 as s_store_name, s_company_name@6 as s_company_name, d_moy@4 as d_moy], aggr=[sum(store_sales.ss_sales_price)] + │ ProjectionExec: expr=[i_brand@2 as i_brand, i_class@3 as i_class, i_category@4 as i_category, ss_sales_price@5 as ss_sales_price, d_moy@6 as d_moy, s_store_name@0 as s_store_name, s_company_name@1 as s_company_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@3, ss_store_sk@3)], projection=[s_store_name@1, s_company_name@2, i_brand@4, i_class@5, i_category@6, ss_sales_price@8, d_moy@9] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[i_brand@1 as i_brand, i_class@2 as i_class, i_category@3 as i_category, ss_store_sk@4 as ss_store_sk, ss_sales_price@5 as ss_sales_price, d_moy@0 as d_moy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@3)], projection=[d_moy@1, i_brand@3, i_class@4, i_category@5, ss_store_sk@7, ss_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand@1, i_class@2, i_category@3, ss_sold_date_sk@4, ss_store_sk@6, ss_sales_price@7] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_name@1 as s_store_name, s_company_name@2 as s_company_name, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_company_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_moy@1 as d_moy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1999, projection=[d_date_sk@0, d_moy@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: (i_category@3 = Books OR i_category@3 = Electronics OR i_category@3 = Sports) AND (i_class@2 = computers OR i_class@2 = stereo OR i_class@2 = football) OR (i_category@3 = Men OR i_category@3 = Jewelry OR i_category@3 = Women) AND (i_class@2 = shirts OR i_class@2 = birdal OR i_class@2 = dresses) + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_class, i_category], file_type=parquet, predicate=(i_category@3 = Books OR i_category@3 = Electronics OR i_category@3 = Sports) AND (i_class@2 = computers OR i_class@2 = stereo OR i_class@2 = football) OR (i_category@3 = Men OR i_category@3 = Jewelry OR i_category@3 = Women) AND (i_class@2 = shirts OR i_class@2 = birdal OR i_class@2 = dresses), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= computers AND computers <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= stereo AND stereo <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= football AND football <= i_class_max@5) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= shirts AND shirts <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= birdal AND birdal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= dresses AND dresses <= i_class_max@5), required_guarantees=[] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_90() -> Result<()> { + let display = test_tpcds_query(90).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortExec: TopK(fetch=100), expr=[am_pm_ratio@0 ASC NULLS LAST], preserve_partitioning=[false] + │ ProjectionExec: expr=[CASE WHEN pmc@1 = 0 THEN None,23,8 ELSE CAST(amc@0 AS Decimal128(15, 4)) / CAST(pmc@1 AS Decimal128(15, 4)) END as am_pm_ratio] + │ CrossJoinExec + │ ProjectionExec: expr=[count(Int64(1))@0 as amc] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_page.wp_web_page_sk AS Float64)@1, ws_web_page_sk@0)] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ws_sold_time_sk@0)], projection=[ws_web_page_sk@3] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ws_ship_hdemo_sk@1)], projection=[ws_sold_time_sk@2, ws_web_page_sk@4] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_time_sk, ws_ship_hdemo_sk, ws_web_page_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[count(Int64(1))@0 as pmc] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_page.wp_web_page_sk AS Float64)@1, ws_web_page_sk@0)] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ws_sold_time_sk@0)], projection=[ws_web_page_sk@3] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ws_ship_hdemo_sk@1)], projection=[ws_sold_time_sk@2, ws_web_page_sk@4] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_time_sk, ws_ship_hdemo_sk, ws_web_page_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[wp_web_page_sk@0 as wp_web_page_sk, CAST(wp_web_page_sk@0 AS Float64) as CAST(web_page.wp_web_page_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: wp_char_count@1 >= 5000 AND wp_char_count@1 <= 5200, projection=[wp_web_page_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-3.parquet]]}, projection=[wp_web_page_sk, wp_char_count], file_type=parquet, predicate=wp_char_count@1 >= 5000 AND wp_char_count@1 <= 5200, pruning_predicate=wp_char_count_null_count@1 != row_count@2 AND wp_char_count_max@0 >= 5000 AND wp_char_count_null_count@1 != row_count@2 AND wp_char_count_min@3 <= 5200, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: t_hour@1 >= 8 AND t_hour@1 <= 9, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour], file_type=parquet, predicate=t_hour@1 >= 8 AND t_hour@1 <= 9, pruning_predicate=t_hour_null_count@1 != row_count@2 AND t_hour_max@0 >= 8 AND t_hour_null_count@1 != row_count@2 AND t_hour_min@3 <= 9, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: hd_dep_count@1 = 6, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count], file_type=parquet, predicate=hd_dep_count@1 = 6, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 6 AND 6 <= hd_dep_count_max@1, required_guarantees=[hd_dep_count in (6)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[wp_web_page_sk@0 as wp_web_page_sk, CAST(wp_web_page_sk@0 AS Float64) as CAST(web_page.wp_web_page_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: wp_char_count@1 >= 5000 AND wp_char_count@1 <= 5200, projection=[wp_web_page_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-3.parquet]]}, projection=[wp_web_page_sk, wp_char_count], file_type=parquet, predicate=wp_char_count@1 >= 5000 AND wp_char_count@1 <= 5200, pruning_predicate=wp_char_count_null_count@1 != row_count@2 AND wp_char_count_max@0 >= 5000 AND wp_char_count_null_count@1 != row_count@2 AND wp_char_count_min@3 <= 5200, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: t_hour@1 >= 19 AND t_hour@1 <= 20, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour], file_type=parquet, predicate=t_hour@1 >= 19 AND t_hour@1 <= 20, pruning_predicate=t_hour_null_count@1 != row_count@2 AND t_hour_max@0 >= 19 AND t_hour_null_count@1 != row_count@2 AND t_hour_min@3 <= 20, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: hd_dep_count@1 = 6, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count], file_type=parquet, predicate=hd_dep_count@1 = 6, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 6 AND 6 <= hd_dep_count_max@1, required_guarantees=[hd_dep_count in (6)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_91() -> Result<()> { + let display = test_tpcds_query(91).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[call_center@0 as call_center, call_center_name@1 as call_center_name, manager@2 as manager, returns_loss@3 as returns_loss] + │ SortPreservingMergeExec: [sum(catalog_returns.cr_net_loss)@4 DESC] + │ SortExec: expr=[returns_loss@3 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[cc_call_center_id@0 as call_center, cc_name@1 as call_center_name, cc_manager@2 as manager, sum(catalog_returns.cr_net_loss)@5 as returns_loss, sum(catalog_returns.cr_net_loss)@5 as sum(catalog_returns.cr_net_loss)] + │ AggregateExec: mode=FinalPartitioned, gby=[cc_call_center_id@0 as cc_call_center_id, cc_name@1 as cc_name, cc_manager@2 as cc_manager, cd_marital_status@3 as cd_marital_status, cd_education_status@4 as cd_education_status], aggr=[sum(catalog_returns.cr_net_loss)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cc_call_center_id@0, cc_name@1, cc_manager@2, cd_marital_status@3, cd_education_status@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cc_call_center_id@0 as cc_call_center_id, cc_name@1 as cc_name, cc_manager@2 as cc_manager, cd_marital_status@4 as cd_marital_status, cd_education_status@5 as cd_education_status], aggr=[sum(catalog_returns.cr_net_loss)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_hdemo_sk@4, CAST(household_demographics.hd_demo_sk AS Float64)@1)], projection=[cc_call_center_id@0, cc_name@1, cc_manager@2, cr_net_loss@3, cd_marital_status@5, cd_education_status@6] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@4, CAST(customer_demographics.cd_demo_sk AS Float64)@3)], projection=[cc_call_center_id@0, cc_name@1, cc_manager@2, cr_net_loss@3, c_current_hdemo_sk@5, cd_marital_status@7, cd_education_status@8] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@6, ca_address_sk@0)], projection=[cc_call_center_id@0, cc_name@1, cc_manager@2, cr_net_loss@3, c_current_cdemo_sk@4, c_current_hdemo_sk@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cr_returning_customer_sk@3, CAST(customer.c_customer_sk AS Float64)@4)], projection=[cc_call_center_id@0, cc_name@1, cc_manager@2, cr_net_loss@4, c_current_cdemo_sk@6, c_current_hdemo_sk@7, c_current_addr_sk@8] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cr_returned_date_sk@3, d_date_sk@0)], projection=[cc_call_center_id@0, cc_name@1, cc_manager@2, cr_returning_customer_sk@4, cr_net_loss@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(call_center.cc_call_center_sk AS Float64)@4, cr_call_center_sk@2)], projection=[cc_call_center_id@1, cc_name@2, cc_manager@3, cr_returned_date_sk@5, cr_returning_customer_sk@6, cr_net_loss@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_call_center_sk, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 11, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 1998 AND d_moy@2 = 11 AND DynamicFilter [ empty ], pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_cdemo_sk@1 as c_current_cdemo_sk, c_current_hdemo_sk@2 as c_current_hdemo_sk, c_current_addr_sk@3 as c_current_addr_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk], file_type=parquet + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ca_gmt_offset@1 = Some(-700),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@1 = Some(-700),4,2 AND DynamicFilter [ empty ], pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-700),4,2 AND Some(-700),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-700),4,2)] + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cd_marital_status@1 = M AND cd_education_status@2 = Unknown OR cd_marital_status@1 = W AND cd_education_status@2 = Advanced Degree + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-3.parquet:..]]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_marital_status@1 = M AND cd_education_status@2 = Unknown OR cd_marital_status@1 = W AND cd_education_status@2 = Advanced Degree, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Unknown AND Unknown <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (Advanced Degree, Unknown), cd_marital_status in (M, W)] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: hd_buy_potential@1 LIKE Unknown%, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_buy_potential], file_type=parquet, predicate=hd_buy_potential@1 LIKE Unknown%, pruning_predicate=hd_buy_potential_null_count@2 != row_count@3 AND hd_buy_potential_min@0 <= Unknowo AND Unknown <= hd_buy_potential_max@1, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[cc_call_center_sk@0 as cc_call_center_sk, cc_call_center_id@1 as cc_call_center_id, cc_name@2 as cc_name, cc_manager@3 as cc_manager, CAST(cc_call_center_sk@0 AS Float64) as CAST(call_center.cc_call_center_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/call_center/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-3.parquet]]}, projection=[cc_call_center_sk, cc_call_center_id, cc_name, cc_manager], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_92() -> Result<()> { + let display = test_tpcds_query(92).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[sum(web_sales.ws_ext_discount_amt)@0 as Excess Discount Amount] + │ GlobalLimitExec: skip=0, fetch=100 + │ AggregateExec: mode=Final, gby=[], aggr=[sum(web_sales.ws_ext_discount_amt)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(web_sales.ws_ext_discount_amt)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@1, i_item_sk@1)], filter=CAST(ws_ext_discount_amt@0 AS Decimal128(30, 15)) > Float64(1.3) * avg(web_sales.ws_ext_discount_amt)@1, projection=[ws_ext_discount_amt@2] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[CAST(1.3 * CAST(avg(web_sales.ws_ext_discount_amt)@1 AS Float64) AS Decimal128(30, 15)) as Float64(1.3) * avg(web_sales.ws_ext_discount_amt), ws_item_sk@0 as ws_item_sk] + │ AggregateExec: mode=FinalPartitioned, gby=[ws_item_sk@0 as ws_item_sk], aggr=[avg(web_sales.ws_ext_discount_amt)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_item_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ws_item_sk@0 as ws_item_sk], aggr=[avg(web_sales.ws_ext_discount_amt)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_ext_discount_amt@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_ext_discount_amt@3, i_item_sk@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@1 as ws_sold_date_sk, ws_ext_discount_amt@2 as ws_ext_discount_amt, i_item_sk@0 as i_item_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_item_sk@0, ws_sold_date_sk@1, ws_ext_discount_amt@3] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-01-27 AND d_date@1 <= 2000-04-26, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-01-27 AND d_date@1 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-01-27 AND d_date@1 <= 2000-04-26, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-01-27 AND d_date@1 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_manufact_id@1 = 350, projection=[i_item_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=i_manufact_id@1 = 350 AND DynamicFilter [ empty ], pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 350 AND 350 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (350)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_93() -> Result<()> { + let display = test_tpcds_query(93).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [sumsales@1 ASC, ss_customer_sk@0 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[sumsales@1 ASC, ss_customer_sk@0 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[ss_customer_sk@0 as ss_customer_sk, sum(t.act_sales)@1 as sumsales] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_customer_sk@0 as ss_customer_sk], aggr=[sum(t.act_sales)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_customer_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_customer_sk@0 as ss_customer_sk], aggr=[sum(t.act_sales)] + │ ProjectionExec: expr=[ss_customer_sk@0 as ss_customer_sk, CASE WHEN sr_return_quantity@3 IS NOT NULL THEN (ss_quantity@1 - sr_return_quantity@3) * CAST(ss_sales_price@2 AS Float64) ELSE ss_quantity@1 * CAST(ss_sales_price@2 AS Float64) END as act_sales] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(reason.r_reason_sk AS Float64)@1, sr_reason_sk@3)], projection=[ss_customer_sk@2, ss_quantity@3, ss_sales_price@4, sr_return_quantity@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_customer_sk@2 as ss_customer_sk, ss_quantity@3 as ss_quantity, ss_sales_price@4 as ss_sales_price, sr_reason_sk@0 as sr_reason_sk, sr_return_quantity@1 as sr_return_quantity] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(sr_item_sk@0, ss_item_sk@0), (sr_ticket_number@2, ss_ticket_number@2)], projection=[sr_reason_sk@1, sr_return_quantity@3, ss_customer_sk@5, ss_quantity@7, ss_sales_price@8] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[r_reason_sk@0 as r_reason_sk, CAST(r_reason_sk@0 AS Float64) as CAST(reason.r_reason_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: r_reason_desc@1 = reason 28, projection=[r_reason_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/reason/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-3.parquet]]}, projection=[r_reason_sk, r_reason_desc], file_type=parquet, predicate=r_reason_desc@1 = reason 28, pruning_predicate=r_reason_desc_null_count@2 != row_count@3 AND r_reason_desc_min@0 <= reason 28 AND reason 28 <= r_reason_desc_max@1, required_guarantees=[r_reason_desc in (reason 28)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_reason_sk, sr_ticket_number, sr_return_quantity], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@0, ss_ticket_number@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_item_sk, ss_customer_sk, ss_ticket_number, ss_quantity, ss_sales_price], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_94() -> Result<()> { + let display = test_tpcds_query(94).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(alias1)@0 as order count, sum(alias2)@1 as total shipping cost, sum(alias3)@2 as total net profit] + │ GlobalLimitExec: skip=0, fetch=100 + │ AggregateExec: mode=Final, gby=[], aggr=[count(alias1), sum(alias2), sum(alias3)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[count(alias1), sum(alias2), sum(alias3)] + │ AggregateExec: mode=FinalPartitioned, gby=[alias1@0 as alias1], aggr=[alias2, alias3] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([alias1@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ws_order_number@0 as alias1], aggr=[alias2, alias3] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(ws_order_number@0, wr_order_number@0)] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(ws_order_number@1, ws_order_number@1)], filter=ws_warehouse_sk@0 != ws_warehouse_sk@1, projection=[ws_order_number@1, ws_ext_ship_cost@2, ws_net_profit@3] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_site.web_site_sk AS Float64)@1, ws_web_site_sk@0)], projection=[ws_warehouse_sk@3, ws_order_number@4, ws_ext_ship_cost@5, ws_net_profit@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer_address.ca_address_sk AS Float64)@1, ws_ship_addr_sk@0)], projection=[ws_web_site_sk@3, ws_warehouse_sk@4, ws_order_number@5, ws_ext_ship_cost@6, ws_net_profit@7] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_ship_date_sk@0)], projection=[ws_ship_addr_sk@3, ws_web_site_sk@4, ws_warehouse_sk@5, ws_order_number@6, ws_ext_ship_cost@7, ws_net_profit@8] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_warehouse_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_order_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[web_site_sk@0 as web_site_sk, CAST(web_site_sk@0 AS Float64) as CAST(web_site.web_site_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: web_company_name@1 = pri, projection=[web_site_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-3.parquet]]}, projection=[web_site_sk, web_company_name], file_type=parquet, predicate=web_company_name@1 = pri, pruning_predicate=web_company_name_null_count@2 != row_count@3 AND web_company_name_min@0 <= pri AND pri <= web_company_name_max@1, required_guarantees=[web_company_name in (pri)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ca_state@1 = IL, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@1 = IL, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IL AND IL <= ca_state_max@1, required_guarantees=[ca_state in (IL)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 1999-02-01 AND d_date@1 <= 1999-04-02, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 1999-02-01 AND d_date@1 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_95() -> Result<()> { + let display = test_tpcds_query(95).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(alias1)@0 as order count, sum(alias2)@1 as total shipping cost, sum(alias3)@2 as total net profit] + │ GlobalLimitExec: skip=0, fetch=100 + │ AggregateExec: mode=Final, gby=[], aggr=[count(alias1), sum(alias2), sum(alias3)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[count(alias1), sum(alias2), sum(alias3)] + │ AggregateExec: mode=FinalPartitioned, gby=[alias1@0 as alias1], aggr=[alias2, alias3] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([alias1@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ws_order_number@0 as alias1], aggr=[alias2, alias3] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(ws_order_number@0, wr_order_number@0)] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(ws_order_number@0, ws_order_number@0)] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_site.web_site_sk AS Float64)@1, ws_web_site_sk@0)], projection=[ws_order_number@3, ws_ext_ship_cost@4, ws_net_profit@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer_address.ca_address_sk AS Float64)@1, ws_ship_addr_sk@0)], projection=[ws_web_site_sk@3, ws_order_number@4, ws_ext_ship_cost@5, ws_net_profit@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_ship_date_sk@0)], projection=[ws_ship_addr_sk@3, ws_web_site_sk@4, ws_order_number@5, ws_ext_ship_cost@6, ws_net_profit@7] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ws_order_number@1, ws_order_number@1)], filter=ws_warehouse_sk@1 != ws_warehouse_sk@0, projection=[ws_order_number@1] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(wr_order_number@0, ws_order_number@0)], projection=[wr_order_number@0] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ws_order_number@1, ws_order_number@1)], filter=ws_warehouse_sk@1 != ws_warehouse_sk@0, projection=[ws_order_number@1] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[web_site_sk@0 as web_site_sk, CAST(web_site_sk@0 AS Float64) as CAST(web_site.web_site_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: web_company_name@1 = pri, projection=[web_site_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-3.parquet]]}, projection=[web_site_sk, web_company_name], file_type=parquet, predicate=web_company_name@1 = pri, pruning_predicate=web_company_name_null_count@2 != row_count@3 AND web_company_name_min@0 <= pri AND pri <= web_company_name_max@1, required_guarantees=[web_company_name in (pri)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ca_state@1 = IL, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@1 = IL, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IL AND IL <= ca_state_max@1, required_guarantees=[ca_state in (IL)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 1999-02-01 AND d_date@1 <= 1999-04-02, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 1999-02-01 AND d_date@1 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_order_number@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_order_number@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([wr_order_number@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_order_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_order_number@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_order_number@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_96() -> Result<()> { + let display = test_tpcds_query(96).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ GlobalLimitExec: skip=0, fetch=100 + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ss_sold_time_sk@0)], projection=[ss_store_sk@3] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_sold_time_sk@2, ss_store_sk@4] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_store_name@1 = ese, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name], file_type=parquet, predicate=s_store_name@1 = ese, pruning_predicate=s_store_name_null_count@2 != row_count@3 AND s_store_name_min@0 <= ese AND ese <= s_store_name_max@1, required_guarantees=[s_store_name in (ese)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: t_hour@1 = 20 AND t_minute@2 >= 30, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@1 = 20 AND t_minute@2 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 20 AND 20 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (20)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: hd_dep_count@1 = 7, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count], file_type=parquet, predicate=hd_dep_count@1 = 7, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 7 AND 7 <= hd_dep_count_max@1, required_guarantees=[hd_dep_count in (7)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_97() -> Result<()> { + let display = test_tpcds_query(97).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[sum(CASE WHEN ssci.customer_sk IS NOT NULL AND csci.customer_sk IS NULL THEN Int64(1) ELSE Int64(0) END)@0 as store_only, sum(CASE WHEN ssci.customer_sk IS NULL AND csci.customer_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END)@1 as catalog_only, sum(CASE WHEN ssci.customer_sk IS NOT NULL AND csci.customer_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END)@2 as store_and_catalog] + │ GlobalLimitExec: skip=0, fetch=100 + │ AggregateExec: mode=Final, gby=[], aggr=[sum(CASE WHEN ssci.customer_sk IS NOT NULL AND csci.customer_sk IS NULL THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN ssci.customer_sk IS NULL AND csci.customer_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN ssci.customer_sk IS NOT NULL AND csci.customer_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(CASE WHEN ssci.customer_sk IS NOT NULL AND csci.customer_sk IS NULL THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN ssci.customer_sk IS NULL AND csci.customer_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN ssci.customer_sk IS NOT NULL AND csci.customer_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END)] + │ ProjectionExec: expr=[customer_sk@1 IS NOT NULL as __common_expr_1, customer_sk@1 as customer_sk, customer_sk@0 as customer_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Full, on=[(customer_sk@0, customer_sk@0), (item_sk@1, item_sk@1)], projection=[customer_sk@0, customer_sk@2] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[cs_bill_customer_sk@0 as customer_sk, cs_item_sk@1 as item_sk] + │ AggregateExec: mode=FinalPartitioned, gby=[cs_bill_customer_sk@0 as cs_bill_customer_sk, cs_item_sk@1 as cs_item_sk], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@0, cs_item_sk@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cs_bill_customer_sk@0 as cs_bill_customer_sk, cs_item_sk@1 as cs_item_sk], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_bill_customer_sk@3, cs_item_sk@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ss_customer_sk@0 as customer_sk, ss_item_sk@1 as item_sk] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_customer_sk@0 as ss_customer_sk, ss_item_sk@1 as ss_item_sk], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_customer_sk@0, ss_item_sk@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_customer_sk@1 as ss_customer_sk, ss_item_sk@0 as ss_item_sk], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_customer_sk@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_98() -> Result<()> { + let display = test_tpcds_query(98).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_category@2 ASC, i_class@3 ASC, i_item_id@0 ASC, i_item_desc@1 ASC, revenueratio@6 ASC] + │ SortExec: expr=[i_category@2 ASC, i_class@3 ASC, i_item_id@0 ASC, i_item_desc@1 ASC, revenueratio@6 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_category@2 as i_category, i_class@3 as i_class, i_current_price@4 as i_current_price, sum(store_sales.ss_ext_sales_price)@5 as itemrevenue, CAST(sum(store_sales.ss_ext_sales_price)@5 AS Float64) * 100 / CAST(sum(sum(store_sales.ss_ext_sales_price)) PARTITION BY [item.i_class] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@6 AS Float64) as revenueratio] + │ WindowAggExec: wdw=[sum(sum(store_sales.ss_ext_sales_price)) PARTITION BY [item.i_class] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(sum(store_sales.ss_ext_sales_price)) PARTITION BY [item.i_class] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Decimal128(27, 2), nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + │ SortExec: expr=[i_class@3 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_class@3], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_category@2 as i_category, i_class@3 as i_class, i_current_price@4 as i_current_price], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, i_category@2, i_class@3, i_current_price@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id, i_item_desc@2 as i_item_desc, i_category@5 as i_category, i_class@4 as i_class, i_current_price@3 as i_current_price], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_ext_sales_price@3, i_item_id@4, i_item_desc@5, i_current_price@6, i_class@7, i_category@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@5 as ss_sold_date_sk, ss_ext_sales_price@6 as ss_ext_sales_price, i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price, i_class@3 as i_class, i_category@4 as i_category] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_item_id@1, i_item_desc@2, i_current_price@3, i_class@4, i_category@5, ss_sold_date_sk@6, ss_ext_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 1999-02-22 AND d_date@1 <= 1999-03-24, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 1999-02-22 AND d_date@1 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_category@5 = Sports OR i_category@5 = Books OR i_category@5 = Home + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@5 = Sports OR i_category@5 = Books OR i_category@5 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_99() -> Result<()> { + let display = test_tpcds_query(99).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [w_substr@0 ASC, sm_type@1 ASC, cc_name_lower@2 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[w_substr@0 ASC, sm_type@1 ASC, cc_name_lower@2 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[w_substr@0 as w_substr, sm_type@1 as sm_type, lower(cc_name@2) as cc_name_lower, sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END)@3 as 30 days, sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(30) AND catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END)@4 as 31-60 days, sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(60) AND catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END)@5 as 61-90 days, sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(90) AND catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END)@6 as 91-120 days, sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)@7 as >120 days] + │ AggregateExec: mode=FinalPartitioned, gby=[w_substr@0 as w_substr, sm_type@1 as sm_type, cc_name@2 as cc_name], aggr=[sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(30) AND catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(60) AND catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(90) AND catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([w_substr@0, sm_type@1, cc_name@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[w_substr@1 as w_substr, sm_type@2 as sm_type, cc_name@3 as cc_name], aggr=[sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(30) AND catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(60) AND catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(90) AND catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)] + │ ProjectionExec: expr=[cs_ship_date_sk@1 - cs_sold_date_sk@0 as __common_expr_1, w_substr@2 as w_substr, sm_type@3 as sm_type, cc_name@4 as cc_name] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_ship_date_sk@1, CAST(date_dim.d_date_sk AS Float64)@1)], projection=[cs_sold_date_sk@0, cs_ship_date_sk@1, w_substr@2, sm_type@3, cc_name@4] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_call_center_sk@2, CAST(call_center.cc_call_center_sk AS Float64)@2)], projection=[cs_sold_date_sk@0, cs_ship_date_sk@1, w_substr@3, sm_type@4, cc_name@6] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_ship_mode_sk@3, CAST(ship_mode.sm_ship_mode_sk AS Float64)@2)], projection=[cs_sold_date_sk@0, cs_ship_date_sk@1, cs_call_center_sk@2, w_substr@4, sm_type@6] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_ship_date_sk@2 as cs_ship_date_sk, cs_call_center_sk@3 as cs_call_center_sk, cs_ship_mode_sk@4 as cs_ship_mode_sk, w_substr@0 as w_substr] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(sq1.w_warehouse_sk AS Float64)@2, cs_warehouse_sk@4)], projection=[w_substr@0, cs_sold_date_sk@3, cs_ship_date_sk@4, cs_call_center_sk@5, cs_ship_mode_sk@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_call_center_sk, cs_ship_mode_sk, cs_warehouse_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[sm_ship_mode_sk@0 as sm_ship_mode_sk, sm_type@1 as sm_type, CAST(sm_ship_mode_sk@0 AS Float64) as CAST(ship_mode.sm_ship_mode_sk AS Float64)] + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/ship_mode/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/ship_mode/part-3.parquet]]}, projection=[sm_ship_mode_sk, sm_type], file_type=parquet + │ ProjectionExec: expr=[cc_call_center_sk@0 as cc_call_center_sk, cc_name@1 as cc_name, CAST(cc_call_center_sk@0 AS Float64) as CAST(call_center.cc_call_center_sk AS Float64)] + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/call_center/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/call_center/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/call_center/part-3.parquet]]}, projection=[cc_call_center_sk, cc_name], file_type=parquet + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[substr(w_warehouse_name@1, 1, 20) as w_substr, w_warehouse_sk@0 as w_warehouse_sk, CAST(w_warehouse_sk@0 AS Float64) as CAST(sq1.w_warehouse_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet]]}, projection=[w_warehouse_sk, w_warehouse_name], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + static INIT_TEST_TPCDS_TABLES: OnceCell<()> = OnceCell::const_new(); + + async fn test_tpcds_query(query_id: usize) -> Result { + let data_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(format!( + "testdata/tpcds/plans_sf{SF}_partitions{PARQUET_PARTITIONS}" + )); + INIT_TEST_TPCDS_TABLES + .get_or_init(|| async { + if !fs::exists(&data_dir).unwrap_or(false) { + tpcds::generate_tpcds_data(&data_dir, SF, PARQUET_PARTITIONS) + .await + .unwrap(); + } + }) + .await; + + let query_sql = tpcds::get_test_tpcds_query(query_id)?; + // Make distributed localhost context to run queries + let (d_ctx, _guard) = start_localhost_context(NUM_WORKERS, DefaultSessionBuilder).await; + let d_ctx = d_ctx + .with_distributed_files_per_task(FILES_PER_TASK)? + .with_distributed_cardinality_effect_task_scale_factor(CARDINALITY_TASK_COUNT_FACTOR)?; + + tpcds::register_tables(&d_ctx, &data_dir).await?; + + let df = d_ctx.sql(&query_sql).await?; + let plan = df.create_physical_plan().await?; + if plan.as_any().is::() { + Ok(display_plan_ascii(plan.as_ref(), false)) + } else { + Ok("".to_string()) + } + } +} From a02666a02859c2fade02ed359cd2f0c35a3b65ed Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Mon, 29 Dec 2025 12:10:44 +0100 Subject: [PATCH 04/27] Split channel resolver in two (#265) * Split channel resolver in two * Simplify WorkerResolverExtension and ChannelResolverExtension * Add default builder to ArrowFlightEndpoint * Add some docs * Listen to clippy * Split get_flight_client_for_url in two * Fix conflicts * Remove unnecessary channel resolver * Improve WorkerResolver docs * Use one ChannelResolver per runtime * Improve error reporting on client connection failure * Add a from_session_builder method for constructing an InMemoryChannelResolver * Add ChannelResolver and WorkerResolver default implementations for Arcs * Add test for default channel resolver * Extend doc comment * Fix typo --- Cargo.lock | 42 +++ Cargo.toml | 1 + benchmarks/cdk/bin/worker.rs | 98 ++---- benchmarks/src/tpch/run.rs | 4 +- cli/Cargo.toml | 2 +- cli/src/main.rs | 79 +---- examples/in_memory_cluster.rs | 33 +- examples/localhost_run.rs | 33 +- examples/localhost_worker.rs | 59 +--- src/channel_resolver_ext.rs | 118 ------- src/distributed_ext.rs | 91 +++++- src/distributed_planner/distributed_config.rs | 34 +-- .../distributed_physical_optimizer_rule.rs | 28 +- src/distributed_planner/plan_annotator.rs | 6 +- src/distributed_planner/task_estimator.rs | 8 +- src/execution_plans/distributed.rs | 6 +- src/execution_plans/network_coalesce.rs | 6 +- src/execution_plans/network_shuffle.rs | 10 +- src/flight_service/do_get.rs | 10 +- src/flight_service/service.rs | 30 +- src/flight_service/session_builder.rs | 22 +- src/lib.rs | 7 +- src/metrics/task_metrics_collector.rs | 7 +- src/metrics/task_metrics_rewriter.rs | 7 +- src/networking/channel_resolver.rs | 289 ++++++++++++++++++ src/networking/mod.rs | 14 + src/networking/worker_resolver.rs | 70 +++++ src/test_utils/in_memory_channel_resolver.rs | 61 ++-- src/test_utils/localhost.rs | 39 +-- tests/custom_config_extension.rs | 5 +- tests/custom_extension_codec.rs | 9 +- tests/error_propagation.rs | 9 +- tests/introspection.rs | 21 +- tests/stateful_execution_plan.rs | 9 +- tests/udfs.rs | 7 +- 35 files changed, 723 insertions(+), 551 deletions(-) delete mode 100644 src/channel_resolver_ext.rs create mode 100644 src/networking/channel_resolver.rs create mode 100644 src/networking/mod.rs create mode 100644 src/networking/worker_resolver.rs diff --git a/Cargo.lock b/Cargo.lock index 0d61ef9c..7db559ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1426,6 +1426,24 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -1862,6 +1880,7 @@ dependencies = [ "hyper-util", "insta", "itertools", + "moka", "object_store", "parquet", "pin-project", @@ -3554,6 +3573,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "moka" +version = "0.12.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3dec6bd31b08944e08b58fd99373893a6c17054d6f3ea5006cc894f4f4eee2a" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + [[package]] name = "native-tls" version = "0.2.14" @@ -4949,6 +4985,12 @@ dependencies = [ "libc", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tempfile" version = "3.23.0" diff --git a/Cargo.toml b/Cargo.toml index 71cce7fe..70f305b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ object_store = "0.12.3" bytes = "1.10.1" pin-project = "1.1.10" tokio-stream = "0.1.17" +moka = { version = "0.12", features = ["sync"] } # integration_tests deps insta = { version = "1.43.1", features = ["filters"], optional = true } diff --git a/benchmarks/cdk/bin/worker.rs b/benchmarks/cdk/bin/worker.rs index d071abae..30bd5d50 100644 --- a/benchmarks/cdk/bin/worker.rs +++ b/benchmarks/cdk/bin/worker.rs @@ -1,23 +1,19 @@ -use arrow_flight::flight_service_client::FlightServiceClient; use async_trait::async_trait; use aws_config::BehaviorVersion; use aws_sdk_ec2::Client as Ec2Client; use axum::{Json, Router, extract::Query, http::StatusCode, routing::get}; -use dashmap::{DashMap, Entry}; use datafusion::common::DataFusionError; use datafusion::common::instant::Instant; use datafusion::common::runtime::SpawnedTask; -use datafusion::execution::{SessionState, SessionStateBuilder}; +use datafusion::execution::SessionStateBuilder; use datafusion::physical_plan::execute_stream; use datafusion::prelude::SessionContext; use datafusion_distributed::{ - ArrowFlightEndpoint, BoxCloneSyncChannel, ChannelResolver, DistributedExt, - DistributedPhysicalOptimizerRule, DistributedSessionBuilder, DistributedSessionBuilderContext, - create_flight_client, display_plan_ascii, + ArrowFlightEndpoint, DistributedExt, DistributedPhysicalOptimizerRule, + DistributedSessionBuilderContext, WorkerResolver, display_plan_ascii, }; use futures::{StreamExt, TryFutureExt}; use log::{error, info, warn}; -use object_store::ObjectStore; use object_store::aws::AmazonS3Builder; use serde::Serialize; use std::collections::HashMap; @@ -27,7 +23,7 @@ use std::sync::atomic::AtomicBool; use std::sync::{Arc, RwLock}; use std::time::Duration; use structopt::StructOpt; -use tonic::transport::{Channel, Server}; +use tonic::transport::Server; use url::Url; #[derive(Serialize)] @@ -44,43 +40,6 @@ struct Cmd { bucket: String, } -#[derive(Clone)] -struct BenchSessionStateBuilder { - s3_url: Url, - s3: Arc, - channel_resolver: Ec2ChannelResolver, -} - -impl BenchSessionStateBuilder { - fn new(s3_url: Url) -> Result> { - let s3 = AmazonS3Builder::from_env() - .with_bucket_name(s3_url.host().unwrap().to_string()) - .build()?; - Ok(Self { - s3_url, - s3: Arc::new(s3), - channel_resolver: Ec2ChannelResolver::new(), - }) - } -} - -#[async_trait] -impl DistributedSessionBuilder for BenchSessionStateBuilder { - async fn build_session_state( - &self, - ctx: DistributedSessionBuilderContext, - ) -> Result { - let state = SessionStateBuilder::new() - .with_default_features() - .with_runtime_env(ctx.runtime_env) - .with_object_store(&self.s3_url, Arc::clone(&self.s3)) - .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) - .with_distributed_channel_resolver(self.channel_resolver.clone()) - .build(); - Ok(state) - } -} - #[tokio::main] async fn main() -> Result<(), Box> { env_logger::builder() @@ -98,15 +57,27 @@ async fn main() -> Result<(), Box> { // Register S3 object store let s3_url = Url::parse(&format!("s3://{}", cmd.bucket))?; - let state_builder = BenchSessionStateBuilder::new(s3_url)?; info!("Building shared SessionContext for the whole lifetime of the HTTP listener..."); - let state = state_builder - .build_session_state(Default::default()) - .await?; + let s3 = Arc::new( + AmazonS3Builder::from_env() + .with_bucket_name(s3_url.host().unwrap().to_string()) + .build()?, + ); + let state = SessionStateBuilder::new() + .with_default_features() + .with_object_store(&s3_url, Arc::clone(&s3) as _) + .with_distributed_worker_resolver(Ec2WorkerResolver::new()) + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + .build(); let ctx = SessionContext::from(state); - let arrow_flight_endpoint = ArrowFlightEndpoint::try_new(state_builder.clone())?; + let arrow_flight_endpoint = + ArrowFlightEndpoint::from_session_builder(move |ctx: DistributedSessionBuilderContext| { + let s3 = s3.clone(); + let s3_url = s3_url.clone(); + async move { Ok(ctx.builder.with_object_store(&s3_url, s3).build()) } + }); let http_server = axum::serve( listener, Router::new().route( @@ -213,9 +184,8 @@ fn err(s: impl Display) -> (StatusCode, String) { } #[derive(Clone)] -struct Ec2ChannelResolver { +struct Ec2WorkerResolver { urls: Arc>>, - channels: Arc>, } async fn background_ec2_worker_resolver(urls: Arc>>) { @@ -273,35 +243,17 @@ async fn background_ec2_worker_resolver(urls: Arc>>) { }); } -impl Ec2ChannelResolver { +impl Ec2WorkerResolver { fn new() -> Self { let urls = Arc::new(RwLock::new(Vec::new())); - let channels = Arc::new(DashMap::new()); tokio::spawn(background_ec2_worker_resolver(urls.clone())); - Self { urls, channels } + Self { urls } } } #[async_trait] -impl ChannelResolver for Ec2ChannelResolver { +impl WorkerResolver for Ec2WorkerResolver { fn get_urls(&self) -> Result, DataFusionError> { Ok(self.urls.read().unwrap().clone()) } - - async fn get_flight_client_for_url( - &self, - url: &Url, - ) -> Result, DataFusionError> { - let channel = match self.channels.entry(url.clone()) { - Entry::Occupied(v) => v.get().clone(), - Entry::Vacant(v) => { - let endpoint = Channel::from_shared(url.to_string()).unwrap(); - let channel = endpoint.connect_lazy(); - let channel = BoxCloneSyncChannel::new(channel); - v.insert(channel.clone()); - channel - } - }; - Ok(create_flight_client(channel)) - } } diff --git a/benchmarks/src/tpch/run.rs b/benchmarks/src/tpch/run.rs index 0eec0c47..63a2680b 100644 --- a/benchmarks/src/tpch/run.rs +++ b/benchmarks/src/tpch/run.rs @@ -43,7 +43,7 @@ use datafusion::physical_plan::display::DisplayableExecutionPlan; use datafusion::physical_plan::{collect, displayable}; use datafusion::prelude::*; use datafusion_distributed::test_utils::localhost::{ - LocalHostChannelResolver, spawn_flight_service, + LocalHostWorkerResolver, spawn_flight_service, }; use datafusion_distributed::{ DistributedExt, DistributedPhysicalOptimizerRule, DistributedSessionBuilder, @@ -143,7 +143,7 @@ impl DistributedSessionBuilder for RunOpt { .with_default_features() .with_config(config) .with_distributed_user_codec(InMemoryCacheExecCodec) - .with_distributed_channel_resolver(LocalHostChannelResolver::new(self.workers.clone())) + .with_distributed_worker_resolver(LocalHostWorkerResolver::new(self.workers.clone())) .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) .with_distributed_option_extension_from_headers::(&ctx.headers)? .with_distributed_files_per_task( diff --git a/cli/Cargo.toml b/cli/Cargo.toml index bd23b27d..65c4339a 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" [dependencies] datafusion = { version = "51" } -datafusion-distributed = { path = "..", features = ["avro"] } +datafusion-distributed = { path = "..", features = ["avro", "integration"] } datafusion-cli = { version = "51", default-features = false } tokio = { version = "1.46.1", features = ["full"] } clap = { version = "4", features = ["derive"] } diff --git a/cli/src/main.rs b/cli/src/main.rs index 7a8e3887..fee4fc45 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -17,8 +17,6 @@ // File mainly copied from https://github.com/apache/datafusion/blob/main/datafusion-cli/src/main.rs -use arrow_flight::flight_service_client::FlightServiceClient; -use async_trait::async_trait; use clap::Parser; use datafusion::common::config_err; use datafusion::config::ConfigOptions; @@ -35,16 +33,14 @@ use datafusion_cli::{ print_format::PrintFormat, print_options::{MaxRows, PrintOptions}, }; -use datafusion_distributed::{ - ArrowFlightEndpoint, BoxCloneSyncChannel, ChannelResolver, DistributedExt, - DistributedPhysicalOptimizerRule, DistributedSessionBuilderContext, create_flight_client, +use datafusion_distributed::test_utils::in_memory_channel_resolver::{ + InMemoryChannelResolver, InMemoryWorkerResolver, }; -use hyper_util::rt::TokioIo; +use datafusion_distributed::{DistributedExt, DistributedPhysicalOptimizerRule}; use std::env; use std::path::Path; use std::process::ExitCode; use std::sync::Arc; -use tonic::transport::{Endpoint, Server}; #[derive(Debug, Parser, PartialEq)] #[clap(author, version, about, long_about= None)] @@ -153,7 +149,8 @@ async fn main_inner() -> Result<()> { .with_config(session_config) .with_runtime_env(runtime_env) .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) - .with_distributed_channel_resolver(InMemoryChannelResolver::new()) + .with_distributed_worker_resolver(InMemoryWorkerResolver::new(16)) + .with_distributed_channel_resolver(InMemoryChannelResolver::default()) .build(); // enable dynamic file query @@ -265,69 +262,3 @@ fn parse_command(command: &str) -> Result { Err("-c flag expects only non empty commands".to_string()) } } - -const DUMMY_URL: &str = "http://localhost:50051"; - -/// [ChannelResolver] implementation that returns gRPC clients baked by an in-memory -/// tokio duplex rather than a TCP connection. -#[derive(Clone)] -struct InMemoryChannelResolver { - channel: FlightServiceClient, -} - -impl InMemoryChannelResolver { - fn new() -> Self { - let (client, server) = tokio::io::duplex(1024 * 1024); - - let mut client = Some(client); - let channel = Endpoint::try_from(DUMMY_URL) - .expect("Invalid dummy URL for building an endpoint. This should never happen") - .connect_with_connector_lazy(tower::service_fn(move |_| { - let client = client - .take() - .expect("Client taken twice. This should never happen"); - async move { Ok::<_, std::io::Error>(TokioIo::new(client)) } - })); - - let this = Self { - channel: create_flight_client(BoxCloneSyncChannel::new(channel)), - }; - let this_clone = this.clone(); - - let endpoint = - ArrowFlightEndpoint::try_new(move |ctx: DistributedSessionBuilderContext| { - let this = this.clone(); - async move { - let builder = SessionStateBuilder::new() - .with_default_features() - .with_distributed_channel_resolver(this) - .with_runtime_env(ctx.runtime_env.clone()); - Ok(builder.build()) - } - }) - .unwrap(); - - tokio::spawn(async move { - Server::builder() - .add_service(endpoint.into_flight_server()) - .serve_with_incoming(tokio_stream::once(Ok::<_, std::io::Error>(server))) - .await - }); - - this_clone - } -} - -#[async_trait] -impl ChannelResolver for InMemoryChannelResolver { - fn get_urls(&self) -> std::result::Result, DataFusionError> { - Ok(vec![url::Url::parse(DUMMY_URL).unwrap(); 16]) // simulate 16 workers - } - - async fn get_flight_client_for_url( - &self, - _: &url::Url, - ) -> std::result::Result, DataFusionError> { - Ok(self.channel.clone()) - } -} diff --git a/examples/in_memory_cluster.rs b/examples/in_memory_cluster.rs index 86691c1e..da1720ba 100644 --- a/examples/in_memory_cluster.rs +++ b/examples/in_memory_cluster.rs @@ -6,8 +6,8 @@ use datafusion::execution::SessionStateBuilder; use datafusion::prelude::{ParquetReadOptions, SessionContext}; use datafusion_distributed::{ ArrowFlightEndpoint, BoxCloneSyncChannel, ChannelResolver, DistributedExt, - DistributedPhysicalOptimizerRule, DistributedSessionBuilderContext, create_flight_client, - display_plan_ascii, + DistributedPhysicalOptimizerRule, DistributedSessionBuilderContext, WorkerResolver, + create_flight_client, display_plan_ascii, }; use futures::TryStreamExt; use hyper_util::rt::TokioIo; @@ -37,6 +37,7 @@ async fn main() -> Result<(), Box> { let state = SessionStateBuilder::new() .with_default_features() + .with_distributed_worker_resolver(InMemoryWorkerResolver) .with_distributed_channel_resolver(InMemoryChannelResolver::new()) .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) .with_distributed_files_per_task(1)? @@ -88,18 +89,12 @@ impl InMemoryChannelResolver { }; let this_clone = this.clone(); - let endpoint = - ArrowFlightEndpoint::try_new(move |ctx: DistributedSessionBuilderContext| { + let endpoint = ArrowFlightEndpoint::from_session_builder( + move |ctx: DistributedSessionBuilderContext| { let this = this.clone(); - async move { - let builder = SessionStateBuilder::new() - .with_default_features() - .with_distributed_channel_resolver(this) - .with_runtime_env(ctx.runtime_env.clone()); - Ok(builder.build()) - } - }) - .unwrap(); + async move { Ok(ctx.builder.with_distributed_channel_resolver(this).build()) } + }, + ); tokio::spawn(async move { Server::builder() @@ -114,10 +109,6 @@ impl InMemoryChannelResolver { #[async_trait] impl ChannelResolver for InMemoryChannelResolver { - fn get_urls(&self) -> Result, DataFusionError> { - Ok(vec![url::Url::parse(DUMMY_URL).unwrap(); 16]) // simulate 16 workers. - } - async fn get_flight_client_for_url( &self, _: &url::Url, @@ -125,3 +116,11 @@ impl ChannelResolver for InMemoryChannelResolver { Ok(self.channel.clone()) } } + +struct InMemoryWorkerResolver; + +impl WorkerResolver for InMemoryWorkerResolver { + fn get_urls(&self) -> Result, DataFusionError> { + Ok(vec![url::Url::parse(DUMMY_URL).unwrap(); 16]) // simulate 16 workers. + } +} diff --git a/examples/localhost_run.rs b/examples/localhost_run.rs index e49b55db..eae9b6cd 100644 --- a/examples/localhost_run.rs +++ b/examples/localhost_run.rs @@ -1,19 +1,15 @@ use arrow::util::pretty::pretty_format_batches; -use arrow_flight::flight_service_client::FlightServiceClient; use async_trait::async_trait; -use dashmap::{DashMap, Entry}; use datafusion::common::DataFusionError; use datafusion::execution::SessionStateBuilder; use datafusion::prelude::{ParquetReadOptions, SessionContext}; use datafusion_distributed::{ - BoxCloneSyncChannel, ChannelResolver, DistributedExt, DistributedPhysicalOptimizerRule, - display_plan_ascii, + DistributedExt, DistributedPhysicalOptimizerRule, WorkerResolver, display_plan_ascii, }; use futures::TryStreamExt; use std::error::Error; use std::sync::Arc; use structopt::StructOpt; -use tonic::transport::Channel; use url::Url; #[derive(StructOpt)] @@ -36,14 +32,13 @@ struct Args { async fn main() -> Result<(), Box> { let args = Args::from_args(); - let localhost_resolver = LocalhostChannelResolver { + let localhost_resolver = LocalhostWorkerResolver { ports: args.cluster_ports, - cached: DashMap::new(), }; let state = SessionStateBuilder::new() .with_default_features() - .with_distributed_channel_resolver(localhost_resolver) + .with_distributed_worker_resolver(localhost_resolver) .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) .with_distributed_files_per_task(1)? .build(); @@ -67,13 +62,12 @@ async fn main() -> Result<(), Box> { } #[derive(Clone)] -struct LocalhostChannelResolver { +struct LocalhostWorkerResolver { ports: Vec, - cached: DashMap>, } #[async_trait] -impl ChannelResolver for LocalhostChannelResolver { +impl WorkerResolver for LocalhostWorkerResolver { fn get_urls(&self) -> Result, DataFusionError> { Ok(self .ports @@ -81,21 +75,4 @@ impl ChannelResolver for LocalhostChannelResolver { .map(|port| Url::parse(&format!("http://localhost:{port}")).unwrap()) .collect()) } - - async fn get_flight_client_for_url( - &self, - url: &Url, - ) -> Result, DataFusionError> { - match self.cached.entry(url.clone()) { - Entry::Occupied(v) => Ok(v.get().clone()), - Entry::Vacant(v) => { - let channel = Channel::from_shared(url.to_string()) - .unwrap() - .connect_lazy(); - let channel = FlightServiceClient::new(BoxCloneSyncChannel::new(channel)); - v.insert(channel.clone()); - Ok(channel) - } - } - } } diff --git a/examples/localhost_worker.rs b/examples/localhost_worker.rs index 920f3cb9..52c723e6 100644 --- a/examples/localhost_worker.rs +++ b/examples/localhost_worker.rs @@ -1,17 +1,8 @@ -use arrow_flight::flight_service_client::FlightServiceClient; -use async_trait::async_trait; -use dashmap::{DashMap, Entry}; -use datafusion::common::{DataFusionError, not_impl_err}; -use datafusion::execution::SessionStateBuilder; -use datafusion_distributed::{ - ArrowFlightEndpoint, BoxCloneSyncChannel, ChannelResolver, DistributedExt, - DistributedSessionBuilderContext, create_flight_client, -}; +use datafusion_distributed::ArrowFlightEndpoint; use std::error::Error; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use structopt::StructOpt; -use tonic::transport::{Channel, Server}; -use url::Url; +use tonic::transport::Server; #[derive(StructOpt)] #[structopt(name = "localhost_worker", about = "A localhost DataFusion worker")] @@ -24,54 +15,10 @@ struct Args { async fn main() -> Result<(), Box> { let args = Args::from_args(); - let localhost_resolver = LocalhostChannelResolver { - cached: DashMap::new(), - }; - - let endpoint = ArrowFlightEndpoint::try_new(move |ctx: DistributedSessionBuilderContext| { - let local_host_resolver = localhost_resolver.clone(); - async move { - Ok(SessionStateBuilder::new() - .with_runtime_env(ctx.runtime_env) - .with_distributed_channel_resolver(local_host_resolver) - .with_default_features() - .build()) - } - })?; - Server::builder() - .add_service(endpoint.into_flight_server()) + .add_service(ArrowFlightEndpoint::default().into_flight_server()) .serve(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), args.port)) .await?; Ok(()) } - -#[derive(Clone)] -struct LocalhostChannelResolver { - cached: DashMap>, -} - -#[async_trait] -impl ChannelResolver for LocalhostChannelResolver { - fn get_urls(&self) -> Result, DataFusionError> { - not_impl_err!("Not implemented") - } - - async fn get_flight_client_for_url( - &self, - url: &Url, - ) -> Result, DataFusionError> { - match self.cached.entry(url.clone()) { - Entry::Occupied(v) => Ok(v.get().clone()), - Entry::Vacant(v) => { - let channel = Channel::from_shared(url.to_string()) - .unwrap() - .connect_lazy(); - let channel = create_flight_client(BoxCloneSyncChannel::new(channel)); - v.insert(channel.clone()); - Ok(channel) - } - } - } -} diff --git a/src/channel_resolver_ext.rs b/src/channel_resolver_ext.rs deleted file mode 100644 index 7d48fb06..00000000 --- a/src/channel_resolver_ext.rs +++ /dev/null @@ -1,118 +0,0 @@ -use crate::DistributedConfig; -use crate::config_extension_ext::set_distributed_option_extension; -use arrow_flight::flight_service_client::FlightServiceClient; -use async_trait::async_trait; -use datafusion::common::exec_err; -use datafusion::error::DataFusionError; -use datafusion::prelude::SessionConfig; -use std::sync::Arc; -use tonic::body::Body; -use url::Url; - -pub(crate) fn set_distributed_channel_resolver( - cfg: &mut SessionConfig, - channel_resolver: impl ChannelResolver + Send + Sync + 'static, -) { - let opts = cfg.options_mut(); - let channel_resolver_ext = ChannelResolverExtension(Arc::new(channel_resolver)); - if let Some(distributed_cfg) = opts.extensions.get_mut::() { - distributed_cfg.__private_channel_resolver = channel_resolver_ext; - } else { - set_distributed_option_extension(cfg, DistributedConfig { - __private_channel_resolver: channel_resolver_ext, - ..Default::default() - }).expect("Calling set_distributed_option_extension with a default DistributedConfig should never fail"); - } -} - -pub(crate) fn get_distributed_channel_resolver( - cfg: &SessionConfig, -) -> Result, DataFusionError> { - let opts = cfg.options(); - let Some(distributed_cfg) = opts.extensions.get::() else { - return exec_err!("ChannelResolver not present in the session config"); - }; - Ok(Arc::clone(&distributed_cfg.__private_channel_resolver.0)) -} - -#[derive(Clone)] -pub(crate) struct ChannelResolverExtension(pub(crate) Arc); - -pub type BoxCloneSyncChannel = tower::util::BoxCloneSyncService< - http::Request, - http::Response, - tonic::transport::Error, ->; - -/// Abstracts networking details so that users can implement their own network resolution -/// mechanism. -/// -/// # Implementation Note -/// -/// When implementing `get_flight_client_for_url`, it is recommended to use the -/// [`create_flight_client`] helper function to ensure clients are configured with -/// appropriate message size limits for internal communication. This helps avoid message -/// size errors when transferring large datasets. -#[async_trait] -pub trait ChannelResolver { - /// Gets all available worker URLs. Used during stage assignment. - fn get_urls(&self) -> Result, DataFusionError>; - /// For a given URL, get an Arrow Flight client for communicating to it. - /// - /// Consider using [`create_flight_client`] to create the client with appropriate - /// default message size limits. - async fn get_flight_client_for_url( - &self, - url: &Url, - ) -> Result, DataFusionError>; -} - -#[async_trait] -impl ChannelResolver for Arc { - fn get_urls(&self) -> Result, DataFusionError> { - self.as_ref().get_urls() - } - - async fn get_flight_client_for_url( - &self, - url: &Url, - ) -> Result, DataFusionError> { - self.as_ref().get_flight_client_for_url(url).await - } -} - -/// Creates a [`FlightServiceClient`] with high default message size limits. -/// -/// This is a convenience function that wraps [`FlightServiceClient::new`] and configures -/// it with `max_decoding_message_size(usize::MAX)` and `max_encoding_message_size(usize::MAX)` -/// to avoid message size limitations for internal communication. -/// -/// Users implementing custom [`ChannelResolver`]s should use this function in their -/// `get_flight_client_for_url` implementations to ensure consistent behavior with built-in -/// implementations. -/// -/// # Example -/// -/// ```rust,ignore -/// use datafusion_distributed::{create_flight_client, BoxCloneSyncChannel, ChannelResolver}; -/// use arrow_flight::flight_service_client::FlightServiceClient; -/// use tonic::transport::Channel; -/// -/// #[async_trait] -/// impl ChannelResolver for MyResolver { -/// async fn get_flight_client_for_url( -/// &self, -/// url: &Url, -/// ) -> Result, DataFusionError> { -/// let channel = Channel::from_shared(url.to_string())?.connect().await?; -/// Ok(create_flight_client(BoxCloneSyncChannel::new(channel))) -/// } -/// } -/// ``` -pub fn create_flight_client( - channel: BoxCloneSyncChannel, -) -> FlightServiceClient { - FlightServiceClient::new(channel) - .max_decoding_message_size(usize::MAX) - .max_encoding_message_size(usize::MAX) -} diff --git a/src/distributed_ext.rs b/src/distributed_ext.rs index 49e797f5..95b20f6c 100644 --- a/src/distributed_ext.rs +++ b/src/distributed_ext.rs @@ -1,10 +1,10 @@ -use crate::channel_resolver_ext::set_distributed_channel_resolver; use crate::config_extension_ext::{ set_distributed_option_extension, set_distributed_option_extension_from_headers, }; use crate::distributed_planner::set_distributed_task_estimator; +use crate::networking::{set_distributed_channel_resolver, set_distributed_worker_resolver}; use crate::protobuf::{set_distributed_user_codec, set_distributed_user_codec_arc}; -use crate::{ChannelResolver, DistributedConfig, TaskEstimator}; +use crate::{ChannelResolver, DistributedConfig, TaskEstimator, WorkerResolver}; use datafusion::common::DataFusionError; use datafusion::config::ConfigExtension; use datafusion::execution::{SessionState, SessionStateBuilder}; @@ -181,8 +181,15 @@ pub trait DistributedExt: Sized { /// Same as [DistributedExt::set_distributed_user_codec] but with a dynamic argument. fn set_distributed_user_codec_arc(&mut self, codec: Arc); - /// Injects a [ChannelResolver] implementation for Distributed DataFusion to resolve worker - /// nodes. When running in distributed mode, setting a [ChannelResolver] is required. + /// This is what tells Distributed DataFusion the URLs of the workers available for serving queries. + /// + /// It injects a [WorkerResolver] implementation for Distributed DataFusion to resolve worker + /// nodes in the cluster. When running in distributed mode, setting a [WorkerResolver] is required. + /// + /// Even if this is required to be present in the [SessionContext] that first initiates and + /// plans the query, it's not necessary to be present in a worker's ArrowFlightEndpoint session + /// state builder, as no planning happens there, and the WorkerResolver::get_urls method is + /// only called during planning. /// /// Example: /// @@ -194,17 +201,59 @@ pub trait DistributedExt: Sized { /// # use datafusion::prelude::SessionConfig; /// # use url::Url; /// # use std::sync::Arc; - /// # use datafusion_distributed::{BoxCloneSyncChannel, ChannelResolver, DistributedExt, DistributedPhysicalOptimizerRule, DistributedSessionBuilderContext}; + /// # use datafusion_distributed::{BoxCloneSyncChannel, WorkerResolver, DistributedExt, DistributedPhysicalOptimizerRule, DistributedSessionBuilderContext}; /// - /// struct CustomChannelResolver; + /// struct CustomWorkerResolver; /// /// #[async_trait] - /// impl ChannelResolver for CustomChannelResolver { + /// impl WorkerResolver for CustomWorkerResolver { /// fn get_urls(&self) -> Result, DataFusionError> { /// todo!() /// } + /// } /// + /// // This tweaks the SessionState so that it can plan for distributed queries and execute them. + /// let state = SessionStateBuilder::new() + /// .with_distributed_worker_resolver(CustomWorkerResolver) + /// // the DistributedPhysicalOptimizerRule also needs to be passed so that query plans + /// // get distributed. + /// .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + /// .build(); + /// ``` + fn with_distributed_worker_resolver( + self, + resolver: T, + ) -> Self; + + /// Same as [DistributedExt::with_distributed_channel_resolver] but with an in-place mutation. + fn set_distributed_worker_resolver( + &mut self, + resolver: T, + ); + + /// This is what tells Distributed DataFusion how to build an Arrow Flight client out of a worker URL. + /// + /// There's a default implementation of this that caches the Arrow Flight client instances so that there's + /// only one per URL, but users can decide to override that behavior in favor of their own solution. + /// + /// Example: + /// + /// ``` + /// # use arrow_flight::flight_service_client::FlightServiceClient; + /// # use async_trait::async_trait; + /// # use datafusion::common::DataFusionError; + /// # use datafusion::execution::{SessionState, SessionStateBuilder}; + /// # use datafusion::prelude::SessionConfig; + /// # use url::Url; + /// # use std::sync::Arc; + /// # use datafusion_distributed::{BoxCloneSyncChannel, ChannelResolver, DistributedExt, DistributedPhysicalOptimizerRule, DistributedSessionBuilderContext}; + /// + /// struct CustomChannelResolver; + /// + /// #[async_trait] + /// impl ChannelResolver for CustomChannelResolver { /// async fn get_flight_client_for_url(&self, url: &Url) -> Result, DataFusionError> { + /// // Build a custom FlightServiceClient wrapped with tower layers or something similar. /// todo!() /// } /// } @@ -221,6 +270,8 @@ pub trait DistributedExt: Sized { /// // part of a plan, it knows how to resolve gRPC channels from URLs for making network calls to other nodes. /// async fn build_state(ctx: DistributedSessionBuilderContext) -> Result { /// Ok(SessionStateBuilder::new() + /// // If you have a custom channel resolver, it should also be passed in the + /// // ArrowFlightEndpoint session builder. /// .with_distributed_channel_resolver(CustomChannelResolver) /// .build()) /// } @@ -393,6 +444,13 @@ impl DistributedExt for SessionConfig { set_distributed_user_codec_arc(self, codec) } + fn set_distributed_worker_resolver( + &mut self, + resolver: T, + ) { + set_distributed_worker_resolver(self, resolver); + } + fn set_distributed_channel_resolver( &mut self, resolver: T, @@ -449,6 +507,10 @@ impl DistributedExt for SessionConfig { #[expr($;self)] fn with_distributed_user_codec_arc(mut self, codec: Arc) -> Self; + #[call(set_distributed_worker_resolver)] + #[expr($;self)] + fn with_distributed_worker_resolver(mut self, resolver: T) -> Self; + #[call(set_distributed_channel_resolver)] #[expr($;self)] fn with_distributed_channel_resolver(mut self, resolver: T) -> Self; @@ -495,6 +557,11 @@ impl DistributedExt for SessionStateBuilder { #[expr($;self)] fn with_distributed_user_codec_arc(mut self, codec: Arc) -> Self; + fn set_distributed_worker_resolver(&mut self, resolver: T); + #[call(set_distributed_worker_resolver)] + #[expr($;self)] + fn with_distributed_worker_resolver(mut self, resolver: T) -> Self; + fn set_distributed_channel_resolver(&mut self, resolver: T); #[call(set_distributed_channel_resolver)] #[expr($;self)] @@ -546,6 +613,11 @@ impl DistributedExt for SessionState { #[expr($;self)] fn with_distributed_user_codec_arc(mut self, codec: Arc) -> Self; + fn set_distributed_worker_resolver(&mut self, resolver: T); + #[call(set_distributed_worker_resolver)] + #[expr($;self)] + fn with_distributed_worker_resolver(mut self, resolver: T) -> Self; + fn set_distributed_channel_resolver(&mut self, resolver: T); #[call(set_distributed_channel_resolver)] #[expr($;self)] @@ -597,6 +669,11 @@ impl DistributedExt for SessionContext { #[expr($;self)] fn with_distributed_user_codec_arc(self, codec: Arc) -> Self; + fn set_distributed_worker_resolver(&mut self, resolver: T); + #[call(set_distributed_worker_resolver)] + #[expr($;self)] + fn with_distributed_worker_resolver(self, resolver: T) -> Self; + fn set_distributed_channel_resolver(&mut self, resolver: T); #[call(set_distributed_channel_resolver)] #[expr($;self)] diff --git a/src/distributed_planner/distributed_config.rs b/src/distributed_planner/distributed_config.rs index d17fc4b2..c85b8e6e 100644 --- a/src/distributed_planner/distributed_config.rs +++ b/src/distributed_planner/distributed_config.rs @@ -1,14 +1,11 @@ -use crate::channel_resolver_ext::ChannelResolverExtension; +use crate::TaskEstimator; use crate::distributed_planner::task_estimator::CombinedTaskEstimator; -use crate::{BoxCloneSyncChannel, ChannelResolver, TaskEstimator}; -use arrow_flight::flight_service_client::FlightServiceClient; -use async_trait::async_trait; +use crate::networking::{ChannelResolverExtension, WorkerResolverExtension}; use datafusion::common::utils::get_available_parallelism; use datafusion::common::{DataFusionError, extensions_options, not_impl_err, plan_err}; use datafusion::config::{ConfigExtension, ConfigField, ConfigOptions, Visit}; use std::fmt::{Debug, Formatter}; use std::sync::Arc; -use url::Url; extensions_options! { /// Configuration for the distributed planner. @@ -40,6 +37,9 @@ extensions_options! { /// [ChannelResolver] implementation that tells the distributed planner information about /// the available workers ready to execute distributed tasks. pub(crate) __private_channel_resolver: ChannelResolverExtension, default = ChannelResolverExtension::default() + /// [WorkerResolver] implementation that tells the distributed planner information about + /// the available workers ready to execute distributed tasks. + pub(crate) __private_worker_resolver: WorkerResolverExtension, default = WorkerResolverExtension::not_implemented() } } @@ -112,31 +112,25 @@ impl ConfigField for ChannelResolverExtension { } } -impl Default for ChannelResolverExtension { - fn default() -> Self { - Self(Arc::new(NotImplementedChannelResolver)) - } -} - impl Debug for ChannelResolverExtension { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "ChannelResolverExtension") } } -struct NotImplementedChannelResolver; +impl ConfigField for WorkerResolverExtension { + fn visit(&self, _: &mut V, _: &str, _: &'static str) { + // nothing to do. + } -#[async_trait] -impl ChannelResolver for NotImplementedChannelResolver { - fn get_urls(&self) -> Result, DataFusionError> { + fn set(&mut self, _: &str, _: &str) -> datafusion::common::Result<()> { not_impl_err!("Not implemented") } +} - async fn get_flight_client_for_url( - &self, - _: &Url, - ) -> Result, DataFusionError> { - not_impl_err!("Not implemented") +impl Debug for WorkerResolverExtension { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "WorkerResolverExtension") } } diff --git a/src/distributed_planner/distributed_physical_optimizer_rule.rs b/src/distributed_planner/distributed_physical_optimizer_rule.rs index 6fa1facf..2f78b83a 100644 --- a/src/distributed_planner/distributed_physical_optimizer_rule.rs +++ b/src/distributed_planner/distributed_physical_optimizer_rule.rs @@ -195,7 +195,7 @@ fn push_down_batch_coalescing( #[cfg(test)] mod tests { - use crate::test_utils::in_memory_channel_resolver::InMemoryChannelResolver; + use crate::test_utils::in_memory_channel_resolver::InMemoryWorkerResolver; use crate::test_utils::parquet::register_parquet_tables; use crate::{DistributedExt, DistributedPhysicalOptimizerRule}; use crate::{assert_snapshot, display_plan_ascii}; @@ -235,7 +235,7 @@ mod tests { SELECT * FROM weather "#; let plan = sql_to_explain(query, |b| { - b.with_distributed_channel_resolver(InMemoryChannelResolver::new(3)) + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) }) .await; assert_snapshot!(plan, @"DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet"); @@ -247,7 +247,7 @@ mod tests { SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*) "#; let plan = sql_to_explain(query, |b| { - b.with_distributed_channel_resolver(InMemoryChannelResolver::new(3)) + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) }) .await; assert_snapshot!(plan, @r" @@ -279,7 +279,7 @@ mod tests { SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*) "#; let plan = sql_to_explain(query, |b| { - b.with_distributed_channel_resolver(InMemoryChannelResolver::new(2)) + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(2)) }) .await; assert_snapshot!(plan, @r" @@ -311,7 +311,7 @@ mod tests { SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*) "#; let plan = sql_to_explain(query, |b| { - b.with_distributed_channel_resolver(InMemoryChannelResolver::new(0)) + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(0)) }) .await; assert_snapshot!(plan, @r" @@ -334,7 +334,7 @@ mod tests { SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*) "#; let plan = sql_to_explain(query, |b| { - b.with_distributed_channel_resolver(InMemoryChannelResolver::new(3)) + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) .with_distributed_cardinality_effect_task_scale_factor(3.0) .unwrap() }) @@ -365,7 +365,7 @@ mod tests { SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*) "#; let plan = sql_to_explain(query, |b| { - b.with_distributed_channel_resolver(InMemoryChannelResolver::new(3)) + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) .with_distributed_files_per_task(3) .unwrap() }) @@ -390,7 +390,7 @@ mod tests { SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*) "#; let plan = sql_to_explain(query, |b| { - b.with_distributed_channel_resolver(InMemoryChannelResolver::new(3)) + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) }) .await; assert_snapshot!(plan, @r" @@ -422,7 +422,7 @@ mod tests { SELECT a."MinTemp", b."MaxTemp" FROM weather a LEFT JOIN weather b ON a."RainToday" = b."RainToday" "#; let plan = sql_to_explain(query, |b| { - b.with_distributed_channel_resolver(InMemoryChannelResolver::new(3)) + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) }) .await; assert_snapshot!(plan, @r" @@ -460,7 +460,7 @@ mod tests { ON a."RainTomorrow" = b."RainTomorrow" "#; let plan = sql_to_explain(query, |b| { - b.with_distributed_channel_resolver(InMemoryChannelResolver::new(3)) + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) }) .await; assert_snapshot!(plan, @r" @@ -508,7 +508,7 @@ mod tests { SELECT * FROM weather ORDER BY "MinTemp" DESC "#; let plan = sql_to_explain(query, |b| { - b.with_distributed_channel_resolver(InMemoryChannelResolver::new(3)) + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) }) .await; assert_snapshot!(plan, @r" @@ -530,7 +530,7 @@ mod tests { SELECT DISTINCT "RainToday", "WindGustDir" FROM weather "#; let plan = sql_to_explain(query, |b| { - b.with_distributed_channel_resolver(InMemoryChannelResolver::new(3)) + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) }) .await; assert_snapshot!(plan, @r" @@ -559,7 +559,7 @@ mod tests { SHOW COLUMNS from weather "#; let plan = sql_to_explain(query, |b| { - b.with_distributed_channel_resolver(InMemoryChannelResolver::new(3)) + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) }) .await; assert_snapshot!(plan, @r" @@ -581,7 +581,7 @@ mod tests { SELECT 1 FROM flights_1m "#; let plan = sql_to_explain(query, |b| { - b.with_distributed_channel_resolver(InMemoryChannelResolver::new(2)) + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(2)) }) .await; assert_snapshot!(plan, @r" diff --git a/src/distributed_planner/plan_annotator.rs b/src/distributed_planner/plan_annotator.rs index f3182bcb..10cae1ca 100644 --- a/src/distributed_planner/plan_annotator.rs +++ b/src/distributed_planner/plan_annotator.rs @@ -136,7 +136,7 @@ fn _annotate_plan( ) -> Result { use TaskCountAnnotation::*; let d_cfg = DistributedConfig::from_config_options(cfg)?; - let n_workers = d_cfg.__private_channel_resolver.0.get_urls()?.len().max(1); + let n_workers = d_cfg.__private_worker_resolver.0.get_urls()?.len().max(1); let annotated_children = plan .children() @@ -303,7 +303,7 @@ fn required_network_boundary_below(parent: &dyn ExecutionPlan) -> Option, ) -> Result { // get the channel manager and current stage from our context - let channel_resolver = get_distributed_channel_resolver(context.session_config())?; + let channel_resolver = get_distributed_channel_resolver(context.as_ref()); let d_cfg = DistributedConfig::from_config_options(context.session_config().options())?; let retrieve_metrics = d_cfg.collect_metrics; diff --git a/src/execution_plans/network_shuffle.rs b/src/execution_plans/network_shuffle.rs index b2198b88..44806f22 100644 --- a/src/execution_plans/network_shuffle.rs +++ b/src/execution_plans/network_shuffle.rs @@ -1,4 +1,3 @@ -use crate::channel_resolver_ext::get_distributed_channel_resolver; use crate::common::require_one_child; use crate::config_extension_ext::ContextGrpcMetadata; use crate::execution_plans::common::{ @@ -7,12 +6,11 @@ use crate::execution_plans::common::{ use crate::flight_service::DoGet; use crate::metrics::MetricsCollectingStream; use crate::metrics::proto::MetricsSetProto; +use crate::networking::get_distributed_channel_resolver; use crate::protobuf::StageKey; use crate::protobuf::{map_flight_to_datafusion_error, map_status_to_datafusion_error}; use crate::stage::{MaybeEncodedPlan, Stage}; -use crate::{ - ChannelResolver, DistributedConfig, DistributedTaskContext, ExecutionTask, NetworkBoundary, -}; +use crate::{DistributedConfig, DistributedTaskContext, ExecutionTask, NetworkBoundary}; use arrow_flight::Ticket; use arrow_flight::decode::FlightRecordBatchStream; use arrow_flight::error::FlightError; @@ -246,7 +244,7 @@ impl ExecutionPlan for NetworkShuffleExec { context: Arc, ) -> Result { // get the channel manager and current stage from our context - let channel_resolver = get_distributed_channel_resolver(context.session_config())?; + let channel_resolver = get_distributed_channel_resolver(context.as_ref()); let d_cfg = DistributedConfig::from_config_options(context.session_config().options())?; let retrieve_metrics = d_cfg.collect_metrics; @@ -266,7 +264,7 @@ impl ExecutionPlan for NetworkShuffleExec { // TODO: this propagation should be automatic https://github.com/datafusion-contrib/datafusion-distributed/issues/247 let context_headers = manually_propagate_distributed_config(context_headers, d_cfg); let stream = input_stage_tasks.into_iter().enumerate().map(|(i, task)| { - let channel_resolver = Arc::clone(&channel_resolver); + let channel_resolver = channel_resolver.clone(); let ticket = Request::from_parts( MetadataMap::from_headers(context_headers.clone()), diff --git a/src/flight_service/do_get.rs b/src/flight_service/do_get.rs index f030f383..6ab6ba06 100644 --- a/src/flight_service/do_get.rs +++ b/src/flight_service/do_get.rs @@ -22,6 +22,7 @@ use datafusion::arrow::array::{Array, AsArray, RecordBatch}; use datafusion::common::exec_datafusion_err; use datafusion::error::DataFusionError; +use datafusion::execution::SessionStateBuilder; use datafusion::physical_plan::ExecutionPlan; use datafusion::prelude::SessionContext; use datafusion_proto::physical_plan::AsExecutionPlan; @@ -80,7 +81,9 @@ impl ArrowFlightEndpoint { let mut session_state = self .session_builder .build_session_state(DistributedSessionBuilderContext { - runtime_env: Arc::clone(&self.runtime), + builder: SessionStateBuilder::new() + .with_default_features() + .with_runtime_env(Arc::clone(&self.runtime)), headers: headers.clone(), }) .await @@ -277,7 +280,6 @@ fn garbage_collect_arrays(batch: RecordBatch) -> Result, } -impl ArrowFlightEndpoint { - pub fn try_new( - session_builder: impl DistributedSessionBuilder + Send + Sync + 'static, - ) -> Result { - let ttl_map = TTLMap::try_new(TTLMapConfig::default())?; - Ok(Self { +impl Default for ArrowFlightEndpoint { + fn default() -> Self { + let ttl_map = TTLMap::try_new(TTLMapConfig::default()) + .expect("Instantiating a TTLMap with default params should never fail"); + Self { runtime: Arc::new(RuntimeEnv::default()), task_data_entries: Arc::new(ttl_map), - session_builder: Arc::new(session_builder), + session_builder: Arc::new(DefaultSessionBuilder), hooks: ArrowFlightEndpointHooks::default(), max_message_size: Some(usize::MAX), - }) + } + } +} + +impl ArrowFlightEndpoint { + /// Builds an [ArrowFlightEndpoint] with a custom [DistributedSessionBuilder]. Use this + /// method whenever you need to add custom stuff to the `SessionContext` that executes the query. + pub fn from_session_builder( + session_builder: impl DistributedSessionBuilder + Send + Sync + 'static, + ) -> Self { + Self { + session_builder: Arc::new(session_builder), + ..Default::default() + } } /// Adds a callback for when an [ExecutionPlan] is received in the `do_get` call. diff --git a/src/flight_service/session_builder.rs b/src/flight_service/session_builder.rs index 1a387bc5..d0afdbf8 100644 --- a/src/flight_service/session_builder.rs +++ b/src/flight_service/session_builder.rs @@ -1,13 +1,12 @@ use async_trait::async_trait; use datafusion::error::DataFusionError; -use datafusion::execution::runtime_env::RuntimeEnv; use datafusion::execution::{SessionState, SessionStateBuilder}; use http::HeaderMap; use std::sync::Arc; -#[derive(Debug, Clone, Default)] +#[derive(Debug, Default)] pub struct DistributedSessionBuilderContext { - pub runtime_env: Arc, + pub builder: SessionStateBuilder, pub headers: HeaderMap, } @@ -49,13 +48,11 @@ pub trait DistributedSessionBuilder { /// #[async_trait] /// impl DistributedSessionBuilder for CustomSessionBuilder { /// async fn build_session_state(&self, ctx: DistributedSessionBuilderContext) -> Result { - /// let mut builder = SessionStateBuilder::new() - /// .with_runtime_env(ctx.runtime_env.clone()) - /// .with_default_features(); - /// builder.set_distributed_user_codec(CustomExecCodec); - /// // Add your UDFs, optimization rules, etc... - /// - /// Ok(builder.build()) + /// Ok(ctx + /// .builder + /// .with_distributed_user_codec(CustomExecCodec) + /// // Add your UDFs, optimization rules, etc... + /// .build()) /// } /// } /// ``` @@ -76,10 +73,7 @@ impl DistributedSessionBuilder for DefaultSessionBuilder { &self, ctx: DistributedSessionBuilderContext, ) -> Result { - Ok(SessionStateBuilder::new() - .with_runtime_env(ctx.runtime_env.clone()) - .with_default_features() - .build()) + Ok(ctx.builder.build()) } } diff --git a/src/lib.rs b/src/lib.rs index 0f304dd0..cf6a7c76 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,5 @@ #![deny(clippy::all)] -mod channel_resolver_ext; mod common; mod config_extension_ext; mod distributed_ext; @@ -10,11 +9,11 @@ mod metrics; mod stage; mod distributed_planner; +mod networking; mod protobuf; #[cfg(any(feature = "integration", test))] pub mod test_utils; -pub use channel_resolver_ext::{BoxCloneSyncChannel, ChannelResolver, create_flight_client}; pub use distributed_ext::DistributedExt; pub use distributed_planner::{ DistributedConfig, DistributedPhysicalOptimizerRule, NetworkBoundary, NetworkBoundaryExt, @@ -29,6 +28,10 @@ pub use flight_service::{ MappedDistributedSessionBuilderExt, }; pub use metrics::rewrite_distributed_plan_with_metrics; +pub use networking::{ + BoxCloneSyncChannel, ChannelResolver, DefaultChannelResolver, WorkerResolver, + create_flight_client, +}; pub use stage::{ DistributedTaskContext, ExecutionTask, Stage, display_plan_ascii, display_plan_graphviz, explain_analyze, diff --git a/src/metrics/task_metrics_collector.rs b/src/metrics/task_metrics_collector.rs index f8f89495..f2870e77 100644 --- a/src/metrics/task_metrics_collector.rs +++ b/src/metrics/task_metrics_collector.rs @@ -119,7 +119,9 @@ mod tests { use crate::execution_plans::DistributedExec; use crate::metrics::proto::metrics_set_proto_to_df; - use crate::test_utils::in_memory_channel_resolver::InMemoryChannelResolver; + use crate::test_utils::in_memory_channel_resolver::{ + InMemoryChannelResolver, InMemoryWorkerResolver, + }; use crate::test_utils::plans::{count_plan_nodes, get_stages_and_stage_keys}; use crate::test_utils::session_context::register_temp_parquet_table; use crate::{DistributedExt, DistributedPhysicalOptimizerRule}; @@ -141,7 +143,8 @@ mod tests { let state = SessionStateBuilder::new() .with_default_features() .with_config(config) - .with_distributed_channel_resolver(InMemoryChannelResolver::new(10)) + .with_distributed_worker_resolver(InMemoryWorkerResolver::new(10)) + .with_distributed_channel_resolver(InMemoryChannelResolver::default()) .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) .with_distributed_task_estimator(2) .with_distributed_metrics_collection(true) diff --git a/src/metrics/task_metrics_rewriter.rs b/src/metrics/task_metrics_rewriter.rs index 1fe070d9..ad011e5b 100644 --- a/src/metrics/task_metrics_rewriter.rs +++ b/src/metrics/task_metrics_rewriter.rs @@ -189,7 +189,9 @@ mod tests { use crate::metrics::rewrite_distributed_plan_with_metrics; use crate::metrics::task_metrics_rewriter::stage_metrics_rewriter; use crate::protobuf::StageKey; - use crate::test_utils::in_memory_channel_resolver::InMemoryChannelResolver; + use crate::test_utils::in_memory_channel_resolver::{ + InMemoryChannelResolver, InMemoryWorkerResolver, + }; use crate::test_utils::metrics::make_test_metrics_set_proto_from_seed; use crate::test_utils::plans::count_plan_nodes; use crate::test_utils::session_context::register_temp_parquet_table; @@ -233,7 +235,8 @@ mod tests { if distributed { builder = builder - .with_distributed_channel_resolver(InMemoryChannelResolver::new(10)) + .with_distributed_worker_resolver(InMemoryWorkerResolver::new(10)) + .with_distributed_channel_resolver(InMemoryChannelResolver::default()) .with_distributed_metrics_collection(true) .unwrap() .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) diff --git a/src/networking/channel_resolver.rs b/src/networking/channel_resolver.rs new file mode 100644 index 00000000..52c4ef6c --- /dev/null +++ b/src/networking/channel_resolver.rs @@ -0,0 +1,289 @@ +use crate::DistributedConfig; +use crate::config_extension_ext::set_distributed_option_extension; +use arrow_flight::flight_service_client::FlightServiceClient; +use async_trait::async_trait; +use datafusion::common::{DataFusionError, config_datafusion_err, exec_datafusion_err}; +use datafusion::execution::TaskContext; +use datafusion::prelude::SessionConfig; +use futures::FutureExt; +use futures::future::Shared; +use std::sync::{Arc, LazyLock}; +use std::time::Duration; +use tonic::body::Body; +use tonic::codegen::BoxFuture; +use tonic::transport::Channel; +use tower::ServiceExt; +use url::Url; + +/// Allows users to customize the way Arrow Flight clients are created. A common use case is to +/// wrap the client with tower layers or schedule it in an IO-specific tokio runtime. +/// +/// There is a default implementation of this trait that should be enough for the most common +/// use-cases. +/// +/// # Implementation Notes +/// - This is called per Arrow Flight request, so implementors of this trait should make sure that +/// clients are reused across method calls instead of building a new Arrow Flight client +/// every time. +/// +/// - When implementing `get_flight_client_for_url`, it is recommended to use the +/// [`create_flight_client`] helper function to ensure clients are configured with +/// appropriate message size limits for internal communication. This helps avoid message +/// size errors when transferring large datasets. +#[async_trait] +pub trait ChannelResolver { + /// For a given URL, get an Arrow Flight client for communicating to it. + /// + /// *WARNING*: This method is called for every Arrow Flight gRPC request, so to not create + /// one client connection for each request, users are required to reuse generated clients. + /// It's recommended to rely on [DefaultChannelResolver] either by delegating method calls + /// to it or by copying the implementation. + /// + /// Consider using [`create_flight_client`] to create the client with appropriate + /// default message size limits. + async fn get_flight_client_for_url( + &self, + url: &Url, + ) -> Result, DataFusionError>; +} + +pub(crate) fn set_distributed_channel_resolver( + cfg: &mut SessionConfig, + channel_resolver: impl ChannelResolver + Send + Sync + 'static, +) { + let opts = cfg.options_mut(); + let channel_resolver = ChannelResolverExtension(Some(Arc::new(channel_resolver))); + if let Some(distributed_cfg) = opts.extensions.get_mut::() { + distributed_cfg.__private_channel_resolver = channel_resolver; + } else { + set_distributed_option_extension(cfg, DistributedConfig { + __private_channel_resolver: channel_resolver, + ..Default::default() + }).expect("Calling set_distributed_option_extension with a default DistributedConfig should never fail"); + } +} + +// Unlike TaskContext, a DataFusion RuntimeEnv does not allow to introduce user-defined extensions. +// For the default implementation of the ChannelResolvers, we cannot inject one DefaultChannelResolver +// per TaskContext, as this holds reference to Tonic channels that must outlive a single TaskContext. +// +// The Tonic channels need to be established and reused under a whole RuntimeEnv scope, not a single +// TaskContext, which forces us to put the default implementation in a static global variable that +// stores and reuses tonic channels per RuntimeEnv's pointer address. +static DEFAULT_CHANNEL_RESOLVER_PER_RUNTIME: LazyLock< + moka::sync::Cache< + /* Arc pointer address */ usize, + /* ChannelResolver that reuses built channels */ Arc, + >, +> = LazyLock::new(|| moka::sync::Cache::builder().max_capacity(256).build()); + +pub(crate) fn get_distributed_channel_resolver( + task_ctx: &TaskContext, +) -> Arc { + let opts = task_ctx.session_config().options(); + if let Some(distributed_cfg) = opts.extensions.get::() { + if let Some(cr) = &distributed_cfg.__private_channel_resolver.0 { + return Arc::clone(cr); + } + } + let runtime_addr = Arc::as_ptr(&task_ctx.runtime_env()) as usize; + DEFAULT_CHANNEL_RESOLVER_PER_RUNTIME + .get_with(runtime_addr, || Arc::new(DefaultChannelResolver::default())) +} + +pub type BoxCloneSyncChannel = tower::util::BoxCloneSyncService< + http::Request, + http::Response, + tonic::transport::Error, +>; + +type ChannelCacheValue = Shared>>; + +#[derive(Clone, Default)] +pub(crate) struct ChannelResolverExtension(Option>); + +/// Default implementation of a [ChannelResolver] that connects to the workers given the URL once +/// and stores the connection instance in a TTI cache. +/// +/// Sane default over which other [ChannelResolver] can be built for better customization of the +/// [FlightServiceClient]s. +#[derive(Clone)] +pub struct DefaultChannelResolver { + cache: Arc>, +} + +impl Default for DefaultChannelResolver { + fn default() -> Self { + Self { + cache: Arc::new( + moka::sync::Cache::builder() + // Use an unrealistic max capacity, just in case there is a logic error on the + // user part that produces an unreasonable amount of URLs. + .max_capacity(64556) + // If a channel has not been used in 5 mins, delete it. + .time_to_idle(Duration::from_secs(5 * 60)) + .build(), + ), + } + } +} + +impl DefaultChannelResolver { + /// Gets the cached [BoxCloneSyncChannel] for the given URL, or builds a new one. + pub async fn get_channel(&self, url: &Url) -> Result { + let channel = self.cache.get_with_by_ref(url, move || { + let url = url.to_string(); + async move { + let endpoint = Channel::from_shared(url.clone()).map_err(|err| { + config_datafusion_err!( + "Invalid URL '{url}' returned by WorkerResolver implementation: {err}" + ) + })?; + let mut channel = endpoint.connect().await.map_err(|err| { + DataFusionError::Context( + format!("{err:?}"), + Box::new(exec_datafusion_err!( + "Error connecting to Distributed DataFusion worker on '{url}': {err}" + )), + ) + })?; + channel.ready().await.map_err(|err| { + DataFusionError::Context( + format!("{err:?}"), + Box::new(exec_datafusion_err!( + "Error waiting for Distributed DataFusion channel to be ready on '{url}': {err}" + )), + ) + })?; + Ok(BoxCloneSyncChannel::new(channel)) + } + .boxed() + .shared() + }); + + channel.await.map_err(|err| { + self.cache.invalidate(url); + DataFusionError::Shared(err) + }) + } +} + +#[async_trait] +impl ChannelResolver for DefaultChannelResolver { + async fn get_flight_client_for_url( + &self, + url: &Url, + ) -> Result, DataFusionError> { + self.get_channel(url).await.map(create_flight_client) + } +} + +#[async_trait] +impl ChannelResolver for Arc { + async fn get_flight_client_for_url( + &self, + url: &Url, + ) -> Result, DataFusionError> { + self.as_ref().get_flight_client_for_url(url).await + } +} + +/// Creates a [`FlightServiceClient`] with high default message size limits. +/// +/// This is a convenience function that wraps [`FlightServiceClient::new`] and configures +/// it with `max_decoding_message_size(usize::MAX)` and `max_encoding_message_size(usize::MAX)` +/// to avoid message size limitations for internal communication. +/// +/// Users implementing custom [`ChannelResolver`]s should use this function in their +/// `get_flight_client_for_url` implementations to ensure consistent behavior with built-in +/// implementations. +/// +/// # Example +/// +/// ```rust,ignore +/// use datafusion_distributed::{create_flight_client, BoxCloneSyncChannel, ChannelResolver}; +/// use arrow_flight::flight_service_client::FlightServiceClient; +/// use tonic::transport::Channel; +/// +/// #[async_trait] +/// impl ChannelResolver for MyResolver { +/// async fn get_flight_client_for_url( +/// &self, +/// url: &Url, +/// ) -> Result, DataFusionError> { +/// let channel = Channel::from_shared(url.to_string())?.connect().await?; +/// Ok(create_flight_client(BoxCloneSyncChannel::new(channel))) +/// } +/// } +/// ``` +pub fn create_flight_client( + channel: BoxCloneSyncChannel, +) -> FlightServiceClient { + FlightServiceClient::new(channel) + .max_decoding_message_size(usize::MAX) + .max_encoding_message_size(usize::MAX) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::DefaultSessionBuilder; + use crate::test_utils::localhost::spawn_flight_service; + use datafusion::common::assert_contains; + use datafusion::common::runtime::SpawnedTask; + use std::error::Error; + use std::time::Instant; + use tokio::net::TcpListener; + + #[tokio::test] + async fn fails_establishing_connection() -> Result<(), Box> { + let (url, _guard) = spawn_http_localhost_worker().await?; + drop(_guard); + let channel_resolver = DefaultChannelResolver::default(); + let err = channel_resolver.get_channel(&url).await.unwrap_err(); + assert_contains!(err.to_string(), "tcp connect error"); + Ok(()) + } + + #[tokio::test] + async fn can_establish_connection() -> Result<(), Box> { + let (url, _guard) = spawn_http_localhost_worker().await?; + let channel_resolver = DefaultChannelResolver::default(); + channel_resolver.get_channel(&url).await?; + Ok(()) + } + + #[tokio::test] + async fn channel_resolve_is_cached() -> Result<(), Box> { + let (url, _guard) = spawn_http_localhost_worker().await?; + let channel_resolver = DefaultChannelResolver::default(); + + let start = Instant::now(); + channel_resolver.get_channel(&url).await?; + let first_call = start.elapsed(); + + let start = Instant::now(); + channel_resolver.get_channel(&url).await?; + let second_call = start.elapsed(); + + assert!(first_call > second_call); + Ok(()) + } + + async fn spawn_http_localhost_worker() -> Result<(Url, SpawnedTask<()>), Box> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + + let port = listener + .local_addr() + .expect("Failed to get local address") + .port(); + + let task = SpawnedTask::spawn(async { + if let Err(err) = spawn_flight_service(DefaultSessionBuilder, listener).await { + panic!("{err}") + } + }); + + Ok((Url::parse(&format!("http://127.0.0.1:{port}"))?, task)) + } +} diff --git a/src/networking/mod.rs b/src/networking/mod.rs new file mode 100644 index 00000000..9756c0c7 --- /dev/null +++ b/src/networking/mod.rs @@ -0,0 +1,14 @@ +mod channel_resolver; +mod worker_resolver; + +pub use channel_resolver::{ + BoxCloneSyncChannel, ChannelResolver, DefaultChannelResolver, create_flight_client, +}; +pub(crate) use channel_resolver::{ + ChannelResolverExtension, get_distributed_channel_resolver, set_distributed_channel_resolver, +}; + +pub use worker_resolver::WorkerResolver; +pub(crate) use worker_resolver::{ + WorkerResolverExtension, get_distributed_worker_resolver, set_distributed_worker_resolver, +}; diff --git a/src/networking/worker_resolver.rs b/src/networking/worker_resolver.rs new file mode 100644 index 00000000..bb52b722 --- /dev/null +++ b/src/networking/worker_resolver.rs @@ -0,0 +1,70 @@ +use crate::DistributedConfig; +use crate::config_extension_ext::set_distributed_option_extension; +use datafusion::common::{DataFusionError, exec_err, not_impl_err}; +use datafusion::prelude::SessionConfig; +use std::sync::Arc; +use url::Url; + +/// Resolves a list of worker URLs in the cluster available for executing parts of the plan. +pub trait WorkerResolver { + /// Gets all available worker URLs in the cluster. Note how this method is not async, which + /// means that any async operation involved in discovering worker URLs must happen on a + /// background thread and be retrieved by this method synchronously. + /// + /// This method will be called in several places during distributed planning: + /// - During task count assignation for the different stages, for determining the size of + /// the cluster and limiting the amount of tasks per stage to Vec.length(). + /// - Right before execution, for lazily assigning worker URLs to the different tasks in the + /// plan. This is done as close to execution in order to have fresh worker URLs as updated + /// as possible. + fn get_urls(&self) -> Result, DataFusionError>; +} + +pub(crate) fn set_distributed_worker_resolver( + cfg: &mut SessionConfig, + worker_resolver: impl WorkerResolver + Send + Sync + 'static, +) { + let opts = cfg.options_mut(); + let worker_resolver = WorkerResolverExtension(Arc::new(worker_resolver)); + if let Some(distributed_cfg) = opts.extensions.get_mut::() { + distributed_cfg.__private_worker_resolver = worker_resolver; + } else { + set_distributed_option_extension(cfg, DistributedConfig { + __private_worker_resolver: worker_resolver, + ..Default::default() + }).expect("Calling set_distributed_option_extension with a default DistributedConfig should never fail"); + } +} + +pub(crate) fn get_distributed_worker_resolver( + cfg: &SessionConfig, +) -> Result, DataFusionError> { + let opts = cfg.options(); + let Some(distributed_cfg) = opts.extensions.get::() else { + return exec_err!("ChannelResolver not present in the session config"); + }; + Ok(Arc::clone(&distributed_cfg.__private_worker_resolver.0)) +} + +#[derive(Clone)] +pub(crate) struct WorkerResolverExtension( + pub(crate) Arc, +); + +impl WorkerResolverExtension { + pub(crate) fn not_implemented() -> Self { + struct NotImplementedWorkerResolver; + impl WorkerResolver for NotImplementedWorkerResolver { + fn get_urls(&self) -> Result, DataFusionError> { + not_impl_err!("WorkerResolver::get_urls() not implemented") + } + } + Self(Arc::new(NotImplementedWorkerResolver)) + } +} + +impl WorkerResolver for Arc { + fn get_urls(&self) -> Result, DataFusionError> { + self.as_ref().get_urls() + } +} diff --git a/src/test_utils/in_memory_channel_resolver.rs b/src/test_utils/in_memory_channel_resolver.rs index 677f55e4..8e61006c 100644 --- a/src/test_utils/in_memory_channel_resolver.rs +++ b/src/test_utils/in_memory_channel_resolver.rs @@ -1,11 +1,11 @@ use crate::{ - ArrowFlightEndpoint, BoxCloneSyncChannel, ChannelResolver, DistributedExt, - DistributedSessionBuilderContext, create_flight_client, + ArrowFlightEndpoint, BoxCloneSyncChannel, ChannelResolver, DefaultSessionBuilder, + DistributedExt, DistributedSessionBuilder, MappedDistributedSessionBuilderExt, WorkerResolver, + create_flight_client, }; use arrow_flight::flight_service_client::FlightServiceClient; use async_trait::async_trait; use datafusion::common::DataFusionError; -use datafusion::execution::SessionStateBuilder; use hyper_util::rt::TokioIo; use tonic::transport::{Endpoint, Server}; @@ -16,11 +16,15 @@ const DUMMY_URL: &str = "http://localhost:50051"; #[derive(Clone)] pub struct InMemoryChannelResolver { channel: FlightServiceClient, - n_workers: usize, } impl InMemoryChannelResolver { - pub fn new(n_workers: usize) -> Self { + /// Build an [InMemoryChannelResolver] with a custom [DistributedSessionBuilder]. + /// This allows you to inject your own DataFusion extensions in the in-memory worker + /// spawned by this method. + pub fn from_session_builder( + builder: impl DistributedSessionBuilder + Send + Sync + 'static, + ) -> Self { let (client, server) = tokio::io::duplex(1024 * 1024); let mut client = Some(client); @@ -35,22 +39,13 @@ impl InMemoryChannelResolver { let this = Self { channel: create_flight_client(BoxCloneSyncChannel::new(channel)), - n_workers, }; let this_clone = this.clone(); - let endpoint = - ArrowFlightEndpoint::try_new(move |ctx: DistributedSessionBuilderContext| { - let this = this.clone(); - async move { - let builder = SessionStateBuilder::new() - .with_default_features() - .with_distributed_channel_resolver(this) - .with_runtime_env(ctx.runtime_env.clone()); - Ok(builder.build()) - } - }) - .unwrap(); + let endpoint = ArrowFlightEndpoint::from_session_builder(builder.map(move |builder| { + let this = this.clone(); + Ok(builder.with_distributed_channel_resolver(this).build()) + })); tokio::spawn(async move { Server::builder() @@ -63,14 +58,14 @@ impl InMemoryChannelResolver { } } -#[async_trait] -impl ChannelResolver for InMemoryChannelResolver { - fn get_urls(&self) -> Result, DataFusionError> { - // Set to a high number so that the distributed planner does not limit the maximum - // spawned tasks to just 1. - Ok(vec![url::Url::parse(DUMMY_URL).unwrap(); self.n_workers]) +impl Default for InMemoryChannelResolver { + fn default() -> Self { + Self::from_session_builder(DefaultSessionBuilder) } +} +#[async_trait] +impl ChannelResolver for InMemoryChannelResolver { async fn get_flight_client_for_url( &self, _: &url::Url, @@ -78,3 +73,21 @@ impl ChannelResolver for InMemoryChannelResolver { Ok(self.channel.clone()) } } + +pub struct InMemoryWorkerResolver { + n_workers: usize, +} + +impl InMemoryWorkerResolver { + pub fn new(n_workers: usize) -> Self { + Self { n_workers } + } +} + +impl WorkerResolver for InMemoryWorkerResolver { + fn get_urls(&self) -> Result, DataFusionError> { + // Set to a high number so that the distributed planner does not limit the maximum + // spawned tasks to just 1. + Ok(vec![url::Url::parse(DUMMY_URL).unwrap(); self.n_workers]) + } +} diff --git a/src/test_utils/localhost.rs b/src/test_utils/localhost.rs index 5c4eaf82..131946a7 100644 --- a/src/test_utils/localhost.rs +++ b/src/test_utils/localhost.rs @@ -1,20 +1,17 @@ use crate::{ - ArrowFlightEndpoint, BoxCloneSyncChannel, ChannelResolver, DistributedExt, - DistributedPhysicalOptimizerRule, DistributedSessionBuilder, DistributedSessionBuilderContext, - MappedDistributedSessionBuilderExt, create_flight_client, + ArrowFlightEndpoint, DistributedExt, DistributedPhysicalOptimizerRule, + DistributedSessionBuilder, DistributedSessionBuilderContext, WorkerResolver, }; -use arrow_flight::flight_service_client::FlightServiceClient; use async_trait::async_trait; use datafusion::common::DataFusionError; use datafusion::common::runtime::JoinSet; use datafusion::execution::SessionStateBuilder; -use datafusion::execution::runtime_env::RuntimeEnv; use datafusion::prelude::SessionContext; use std::error::Error; use std::sync::Arc; use std::time::Duration; use tokio::net::TcpListener; -use tonic::transport::{Channel, Server}; +use tonic::transport::Server; use url::Url; /// Create workers and context on localhost with a fixed number of target partitions. @@ -50,14 +47,6 @@ where }) .collect(); - let channel_resolver = LocalHostChannelResolver::new(ports.clone()); - let session_builder = session_builder.map(move |builder: SessionStateBuilder| { - let channel_resolver = channel_resolver.clone(); - Ok(builder - .with_distributed_channel_resolver(channel_resolver) - .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) - .build()) - }); let mut join_set = JoinSet::new(); for listener in listeners { let session_builder = session_builder.clone(); @@ -69,9 +58,13 @@ where } tokio::time::sleep(Duration::from_millis(100)).await; + let worker_resolver = LocalHostWorkerResolver::new(ports); let mut state = session_builder .build_session_state(DistributedSessionBuilderContext { - runtime_env: Arc::new(RuntimeEnv::default()), + builder: SessionStateBuilder::new() + .with_default_features() + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + .with_distributed_worker_resolver(worker_resolver), headers: Default::default(), }) .await @@ -82,11 +75,11 @@ where } #[derive(Clone)] -pub struct LocalHostChannelResolver { +pub struct LocalHostWorkerResolver { ports: Vec, } -impl LocalHostChannelResolver { +impl LocalHostWorkerResolver { pub fn new, I: IntoIterator>(ports: I) -> Self where N::Error: std::fmt::Debug, @@ -98,7 +91,7 @@ impl LocalHostChannelResolver { } #[async_trait] -impl ChannelResolver for LocalHostChannelResolver { +impl WorkerResolver for LocalHostWorkerResolver { fn get_urls(&self) -> Result, DataFusionError> { self.ports .iter() @@ -106,21 +99,13 @@ impl ChannelResolver for LocalHostChannelResolver { .map(|url| Url::parse(&url).map_err(external_err)) .collect::, _>>() } - async fn get_flight_client_for_url( - &self, - url: &Url, - ) -> Result, DataFusionError> { - let endpoint = Channel::from_shared(url.to_string()).map_err(external_err)?; - let channel = endpoint.connect().await.map_err(external_err)?; - Ok(create_flight_client(BoxCloneSyncChannel::new(channel))) - } } pub async fn spawn_flight_service( session_builder: impl DistributedSessionBuilder + Send + Sync + 'static, incoming: TcpListener, ) -> Result<(), Box> { - let endpoint = ArrowFlightEndpoint::try_new(session_builder)?; + let endpoint = ArrowFlightEndpoint::from_session_builder(session_builder); let incoming = tokio_stream::wrappers::TcpListenerStream::new(incoming); diff --git a/tests/custom_config_extension.rs b/tests/custom_config_extension.rs index 1278a948..9d4356c0 100644 --- a/tests/custom_config_extension.rs +++ b/tests/custom_config_extension.rs @@ -26,9 +26,8 @@ mod tests { async fn build_state( ctx: DistributedSessionBuilderContext, ) -> Result { - Ok(SessionStateBuilder::new() - .with_runtime_env(ctx.runtime_env) - .with_default_features() + Ok(ctx + .builder .with_distributed_option_extension_from_headers::(&ctx.headers)? .with_distributed_user_codec(CustomConfigExtensionRequiredExecCodec) .build()) diff --git a/tests/custom_extension_codec.rs b/tests/custom_extension_codec.rs index c92f9b2c..3ef49fe0 100644 --- a/tests/custom_extension_codec.rs +++ b/tests/custom_extension_codec.rs @@ -3,9 +3,7 @@ mod tests { use datafusion::arrow::util::pretty::pretty_format_batches; use datafusion::common::tree_node::{Transformed, TreeNode}; use datafusion::error::DataFusionError; - use datafusion::execution::{ - SendableRecordBatchStream, SessionState, SessionStateBuilder, TaskContext, - }; + use datafusion::execution::{SendableRecordBatchStream, SessionState, TaskContext}; use datafusion::physical_expr::EquivalenceProperties; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::{ @@ -30,9 +28,8 @@ mod tests { async fn build_state( ctx: DistributedSessionBuilderContext, ) -> Result { - Ok(SessionStateBuilder::new() - .with_runtime_env(ctx.runtime_env) - .with_default_features() + Ok(ctx + .builder .with_distributed_user_codec(CustomPassThroughExecCodec) .build()) } diff --git a/tests/error_propagation.rs b/tests/error_propagation.rs index dda7049e..cf042481 100644 --- a/tests/error_propagation.rs +++ b/tests/error_propagation.rs @@ -2,9 +2,7 @@ mod tests { use datafusion::common::tree_node::{Transformed, TreeNode}; use datafusion::error::DataFusionError; - use datafusion::execution::{ - SendableRecordBatchStream, SessionState, SessionStateBuilder, TaskContext, - }; + use datafusion::execution::{SendableRecordBatchStream, SessionState, TaskContext}; use datafusion::physical_expr::EquivalenceProperties; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; @@ -29,9 +27,8 @@ mod tests { async fn build_state( ctx: DistributedSessionBuilderContext, ) -> Result { - Ok(SessionStateBuilder::new() - .with_runtime_env(ctx.runtime_env) - .with_default_features() + Ok(ctx + .builder .with_distributed_user_codec(ErrorThrowingExecCodec) .build()) } diff --git a/tests/introspection.rs b/tests/introspection.rs index 759c8e8c..3a7a5a8a 100644 --- a/tests/introspection.rs +++ b/tests/introspection.rs @@ -1,28 +1,25 @@ #[cfg(all(feature = "integration", test))] mod tests { use datafusion::arrow::util::pretty::pretty_format_batches; - use datafusion::execution::SessionStateBuilder; use datafusion::physical_plan::execute_stream; - use datafusion::prelude::SessionConfig; use datafusion_distributed::test_utils::localhost::start_localhost_context; use datafusion_distributed::test_utils::parquet::register_parquet_tables; use datafusion_distributed::{ - DefaultSessionBuilder, MappedDistributedSessionBuilderExt, assert_snapshot, - display_plan_ascii, + DistributedSessionBuilderContext, assert_snapshot, display_plan_ascii, }; use futures::TryStreamExt; use std::error::Error; #[tokio::test] async fn distributed_show_columns() -> Result<(), Box> { - let (ctx, _guard) = start_localhost_context( - 3, - DefaultSessionBuilder.map(|mut v: SessionStateBuilder| { - v = v.with_config(SessionConfig::default().with_information_schema(true)); - Ok(v.build()) - }), - ) - .await; + let (ctx, _guard) = + start_localhost_context(3, |mut ctx: DistributedSessionBuilderContext| async { + let cfg = ctx.builder.config().get_or_insert_default(); + let opts = cfg.options_mut(); + opts.catalog.information_schema = true; + Ok(ctx.builder.build()) + }) + .await; register_parquet_tables(&ctx).await?; let df = ctx.sql(r#"SHOW COLUMNS from weather"#).await?; diff --git a/tests/stateful_execution_plan.rs b/tests/stateful_execution_plan.rs index 2468c6c4..e6ec725f 100644 --- a/tests/stateful_execution_plan.rs +++ b/tests/stateful_execution_plan.rs @@ -3,9 +3,7 @@ mod tests { use datafusion::arrow::util::pretty::pretty_format_batches; use datafusion::common::tree_node::{Transformed, TreeNode}; use datafusion::error::DataFusionError; - use datafusion::execution::{ - SendableRecordBatchStream, SessionState, SessionStateBuilder, TaskContext, - }; + use datafusion::execution::{SendableRecordBatchStream, SessionState, TaskContext}; use datafusion::physical_expr::EquivalenceProperties; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; @@ -40,9 +38,8 @@ mod tests { async fn build_state( ctx: DistributedSessionBuilderContext, ) -> Result { - Ok(SessionStateBuilder::new() - .with_runtime_env(ctx.runtime_env) - .with_default_features() + Ok(ctx + .builder .with_distributed_user_codec(PassThroughExecCodec) .build()) } diff --git a/tests/udfs.rs b/tests/udfs.rs index dd96beaa..e99c5bef 100644 --- a/tests/udfs.rs +++ b/tests/udfs.rs @@ -4,7 +4,7 @@ mod tests { use arrow::util::pretty::pretty_format_batches; use datafusion::arrow::datatypes::DataType; use datafusion::error::DataFusionError; - use datafusion::execution::{SessionState, SessionStateBuilder}; + use datafusion::execution::SessionState; use datafusion::logical_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility, }; @@ -29,9 +29,8 @@ mod tests { async fn build_state( ctx: DistributedSessionBuilderContext, ) -> Result { - Ok(SessionStateBuilder::new() - .with_runtime_env(ctx.runtime_env) - .with_default_features() + Ok(ctx + .builder .with_scalar_functions(vec![udf()]) .with_distributed_task_estimator(2) .build()) From 69da7db68eadc1ca8e7571c107ebf8e9c12ccf72 Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Mon, 29 Dec 2025 12:43:06 +0100 Subject: [PATCH 05/27] Refactor benchmarks crate and add TPC-DS benchmarks (#269) * Split channel resolver in two * Simplify WorkerResolverExtension and ChannelResolverExtension * Add default builder to ArrowFlightEndpoint * Add some docs * Listen to clippy * Split get_flight_client_for_url in two * Fix conflicts * Remove unnecessary channel resolver * Improve WorkerResolver docs * Use one ChannelResolver per runtime * Improve error reporting on client connection failure * Add a from_session_builder method for constructing an InMemoryChannelResolver * Add ChannelResolver and WorkerResolver default implementations for Arcs * Make TPC-DS tests use DataFusion test dataset * Remove non-working in-memory option from benchmarks * Remove unnecessary utils folder * Refactor benchmark folder * Rename to prepare_tpch.rs * Adapt benchmarks for TPC-DS * Update benchmarks README.md * Fix conflicts * Use default session state builder * Update benchmarks README.md * Make gen-tpcds.sh executable --- benchmarks/Cargo.toml | 2 +- benchmarks/README.md | 77 +-- benchmarks/gen-tpcds.sh | 21 + benchmarks/gen-tpch.sh | 2 +- benchmarks/run.sh | 6 +- benchmarks/src/bin/dfbench.rs | 43 -- benchmarks/src/lib.rs | 2 - benchmarks/src/main.rs | 32 + benchmarks/src/prepare_tpcds.rs | 28 + .../src/{tpch/convert.rs => prepare_tpch.rs} | 77 +-- benchmarks/src/run.rs | 621 ++++++++++++++++++ benchmarks/src/tpch/mod.rs | 188 ------ benchmarks/src/tpch/run.rs | 439 ------------- benchmarks/src/util/memory.rs | 215 ------ benchmarks/src/util/mod.rs | 25 - benchmarks/src/util/options.rs | 154 ----- benchmarks/src/util/run.rs | 270 -------- src/test_utils/tpch.rs | 19 +- tests/tpcds_correctness_test.rs | 188 +++--- tests/tpch_correctness_test.rs | 49 +- tests/tpch_explain_analyze.rs | 7 +- tests/tpch_plans_test.rs | 7 +- 22 files changed, 871 insertions(+), 1601 deletions(-) create mode 100755 benchmarks/gen-tpcds.sh delete mode 100644 benchmarks/src/bin/dfbench.rs delete mode 100644 benchmarks/src/lib.rs create mode 100644 benchmarks/src/main.rs create mode 100644 benchmarks/src/prepare_tpcds.rs rename benchmarks/src/{tpch/convert.rs => prepare_tpch.rs} (62%) create mode 100644 benchmarks/src/run.rs delete mode 100644 benchmarks/src/tpch/mod.rs delete mode 100644 benchmarks/src/tpch/run.rs delete mode 100644 benchmarks/src/util/memory.rs delete mode 100644 benchmarks/src/util/mod.rs delete mode 100644 benchmarks/src/util/options.rs delete mode 100644 benchmarks/src/util/run.rs diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 77f481a9..204943e3 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -30,7 +30,7 @@ aws-sdk-ec2 = "1" [[bin]] name = "dfbench" -path = "src/bin/dfbench.rs" +path = "src/main.rs" [[bin]] name = "worker" diff --git a/benchmarks/README.md b/benchmarks/README.md index e411e900..e85f84d1 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,81 +1,36 @@ # Distributed DataFusion Benchmarks -### Generating TPCH data +### Generating Benchmarking data Generate TPCH data into the `data/` dir ```shell ./gen-tpch.sh +./gen-tpcds.sh ``` -### Running TPCH benchmarks in single-node mode +### Running Benchmarks in single-node mode -After generating the data with the command above, the benchmarks can be run with +After generating the data with the command above, the benchmarks can be run with: ```shell -cargo run -p datafusion-distributed-benchmarks --release -- tpch +WORKERS=0 ./benchmarks/run.sh --threads 2 --path benchmarks/data/tpch_sf1 ``` -For preloading the TPCH data in-memory, the `-m` flag can be passed +- `--threads`: This is the physical threads that the Tokio runtime will use for executing the binary. + It's recommended to set `--threads` to something small, like `2`, for throttling each individual + process running queries, and simulate how adding throttled workers can speed up the queries. +- `--path`: It can point to any folder containing benchmark datasets. -```shell -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m -``` - -For running the benchmarks with using just a specific amount of physical threads: - -```shell -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m --threads 3 -``` - -### Running TPCH benchmarks in distributed mode - -Running the benchmarks in distributed mode implies: - -- running 1 or more workers in separate terminals -- running the benchmarks in an additional terminal - -The workers can be spawned by passing the `--spawn ` flag, for example, for spawning 3 workers: - -```shell -cargo run -p datafusion-distributed-benchmarks --release -- tpch --spawn 8000 -``` - -```shell -cargo run -p datafusion-distributed-benchmarks --release -- tpch --spawn 8001 -``` - -```shell -cargo run -p datafusion-distributed-benchmarks --release -- tpch --spawn 8002 -``` - -With the three workers running in separate terminals, the TPCH benchmarks can be run in distributed mode with: - -```shell -cargo run -p datafusion-distributed-benchmarks --release -- tpch --workers 8000,8001,8002 -``` +### Running Benchmarks benchmarks in distributed mode -A good way of measuring the impact of distribution is to limit the physical threads each worker can use. For example, -it's expected that running 8 workers with 2 physical threads each one (8 * 2 = 16 total) is faster than running in -single-node with just 2 threads (1 * 3 = 2 total). +The same script is used for running distributed benchmarks: ```shell -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m --threads 2 --spawn 8000 & -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m --threads 2 --spawn 8001 & -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m --threads 2 --spawn 8002 & -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m --threads 2 --spawn 8003 & -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m --threads 2 --spawn 8004 & -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m --threads 2 --spawn 8005 & -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m --threads 2 --spawn 8006 & -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m --threads 2 --spawn 8007 & +WORKERS=8 ./benchmarks/run.sh --threads 2 --path ./benchmarks/data/tpch_sf1 --files-per-task 2 ``` -```shell -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m --threads 2 --workers 8000,8001,8002,8003,8004,8005,8006,8007 -``` - -The `run.sh` script already does this for you in a more ergonomic way: - -```shell -WORKERS=8 run.sh --threads 2 -m -``` \ No newline at end of file +- `WORKERS`: Env variable that sets the amount of localhost workers used in the query. +- `--threads`: Sets the Tokio runtime threads for each individual worker and for the benchmarking binary. +- `--path`: It can point to any folder containing benchmark datasets. +- `--files-per-task`: How many files each distributed task will handle. diff --git a/benchmarks/gen-tpcds.sh b/benchmarks/gen-tpcds.sh new file mode 100755 index 00000000..3a41b40b --- /dev/null +++ b/benchmarks/gen-tpcds.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -e + +SCALE_FACTOR=${SCALE_FACTOR:-1} +PARTITIONS=${PARTITIONS:-16} + +echo "Generating TPC-DS dataset with SCALE_FACTOR=${SCALE_FACTOR} and PARTITIONS=${PARTITIONS}" + +# https://stackoverflow.com/questions/59895/how-do-i-get-the-directory-where-a-bash-script-is-located-from-within-the-script +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +DATA_DIR=${DATA_DIR:-$SCRIPT_DIR/data} +CARGO_COMMAND=${CARGO_COMMAND:-"cargo run --release"} +TPCDS_DIR="${DATA_DIR}/tpcds_sf${SCALE_FACTOR}" + +echo "Creating tpcds dataset at Scale Factor ${SCALE_FACTOR} in ${TPCDS_DIR}..." + +# Ensure the target data directory exists +mkdir -p "${TPCDS_DIR}" + +$CARGO_COMMAND -- prepare-tpcds --output "${TPCDS_DIR}" --partitions "$PARTITIONS" diff --git a/benchmarks/gen-tpch.sh b/benchmarks/gen-tpch.sh index 91a23cea..4a26f8db 100755 --- a/benchmarks/gen-tpch.sh +++ b/benchmarks/gen-tpch.sh @@ -33,6 +33,6 @@ if test -d "${FILE}"; then else echo " creating parquet files using benchmark binary ..." pushd "${SCRIPT_DIR}" > /dev/null - $CARGO_COMMAND -- tpch-convert --input "${TPCH_DIR}" --output "${TPCH_DIR}" --format parquet --partitions "$PARTITIONS" + $CARGO_COMMAND -- prepare-tpch --input "${TPCH_DIR}" --output "${TPCH_DIR}" --partitions "$PARTITIONS" popd > /dev/null fi diff --git a/benchmarks/run.sh b/benchmarks/run.sh index 7cb0151a..e97f69b2 100755 --- a/benchmarks/run.sh +++ b/benchmarks/run.sh @@ -8,7 +8,7 @@ WORKERS=${WORKERS:-8} SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) if [ "$WORKERS" == "0" ]; then - cargo run -p datafusion-distributed-benchmarks --release -- tpch "$@" + cargo run -p datafusion-distributed-benchmarks --release -- run "$@" exit fi @@ -38,7 +38,7 @@ cargo build -p datafusion-distributed-benchmarks --release trap cleanup EXIT INT TERM for i in $(seq 0 $((WORKERS-1))); do - "$SCRIPT_DIR"/../target/release/dfbench tpch --spawn $((8000+i)) "$@" & + "$SCRIPT_DIR"/../target/release/dfbench run --spawn $((8000+i)) "$@" & done echo "Waiting for worker ports to be ready..." @@ -46,4 +46,4 @@ for i in $(seq 0 $((WORKERS-1))); do wait_for_port $((8000+i)) done -"$SCRIPT_DIR"/../target/release/dfbench tpch --workers $(seq -s, 8000 $((8000+WORKERS-1))) "$@" +"$SCRIPT_DIR"/../target/release/dfbench run --workers $(seq -s, 8000 $((8000+WORKERS-1))) "$@" diff --git a/benchmarks/src/bin/dfbench.rs b/benchmarks/src/bin/dfbench.rs deleted file mode 100644 index d626f79e..00000000 --- a/benchmarks/src/bin/dfbench.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! DataFusion Distributed benchmark runner -use datafusion::error::Result; - -use structopt::StructOpt; - -use datafusion_distributed_benchmarks::tpch; - -#[derive(Debug, StructOpt)] -#[structopt(about = "benchmark command")] -enum Options { - Tpch(tpch::RunOpt), - TpchConvert(tpch::ConvertOpt), -} - -// Main benchmark runner entrypoint -pub fn main() -> Result<()> { - env_logger::init(); - - match Options::from_args() { - Options::Tpch(opt) => opt.run(), - Options::TpchConvert(opt) => { - let rt = tokio::runtime::Runtime::new()?; - rt.block_on(async { opt.run().await }) - } - } -} diff --git a/benchmarks/src/lib.rs b/benchmarks/src/lib.rs deleted file mode 100644 index 5ede4af0..00000000 --- a/benchmarks/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod tpch; -mod util; diff --git a/benchmarks/src/main.rs b/benchmarks/src/main.rs new file mode 100644 index 00000000..1b60e1c6 --- /dev/null +++ b/benchmarks/src/main.rs @@ -0,0 +1,32 @@ +//! DataFusion Distributed benchmark runner +mod prepare_tpcds; +mod prepare_tpch; +mod run; + +use datafusion::error::Result; +use structopt::StructOpt; + +#[derive(Debug, StructOpt)] +#[structopt(about = "benchmark command")] +enum Options { + Run(run::RunOpt), + PrepareTpch(prepare_tpch::PrepareTpchOpt), + PrepareTpcds(prepare_tpcds::PrepareTpcdsOpt), +} + +// Main benchmark runner entrypoint +pub fn main() -> Result<()> { + env_logger::init(); + + match Options::from_args() { + Options::Run(opt) => opt.run(), + Options::PrepareTpch(opt) => { + let rt = tokio::runtime::Runtime::new()?; + rt.block_on(async { opt.run().await }) + } + Options::PrepareTpcds(opt) => { + let rt = tokio::runtime::Runtime::new()?; + rt.block_on(async { opt.run().await }) + } + } +} diff --git a/benchmarks/src/prepare_tpcds.rs b/benchmarks/src/prepare_tpcds.rs new file mode 100644 index 00000000..b6c10348 --- /dev/null +++ b/benchmarks/src/prepare_tpcds.rs @@ -0,0 +1,28 @@ +use datafusion::error::DataFusionError; +use datafusion_distributed::test_utils::tpcds; +use std::path::{Path, PathBuf}; +use structopt::StructOpt; + +/// Prepare TPC-DS parquet files for benchmarks +#[derive(Debug, StructOpt)] +pub struct PrepareTpcdsOpt { + /// Output path + #[structopt(parse(from_os_str), required = true, short = "o", long = "output")] + output_path: PathBuf, + + /// Number of partitions to produce. By default, uses only 1 partition. + #[structopt(short = "n", long = "partitions", default_value = "1")] + partitions: usize, + + /// Scale factor of the TPC-DS data + #[structopt(long, default_value = "1")] + sf: f64, +} + +impl PrepareTpcdsOpt { + pub async fn run(self) -> datafusion::common::Result<()> { + tpcds::generate_tpcds_data(Path::new(&self.output_path), self.sf, self.partitions) + .await + .map_err(|e| DataFusionError::Internal(format!("{e:?}"))) + } +} diff --git a/benchmarks/src/tpch/convert.rs b/benchmarks/src/prepare_tpch.rs similarity index 62% rename from benchmarks/src/tpch/convert.rs rename to benchmarks/src/prepare_tpch.rs index 6ef59a4c..826ed56a 100644 --- a/benchmarks/src/tpch/convert.rs +++ b/benchmarks/src/prepare_tpch.rs @@ -16,23 +16,20 @@ // under the License. use datafusion::common::instant::Instant; -use datafusion::logical_expr::select_expr::SelectExpr; -use std::fs; -use std::path::{Path, PathBuf}; - -use super::TPCH_TABLES; -use super::get_tbl_tpch_table_schema; -use datafusion::common::not_impl_err; use datafusion::error::Result; +use datafusion::logical_expr::select_expr::SelectExpr; use datafusion::parquet::basic::Compression; use datafusion::parquet::file::properties::WriterProperties; use datafusion::prelude::*; +use datafusion_distributed::test_utils::tpch; +use std::fs; +use std::path::{Path, PathBuf}; use structopt::StructOpt; -/// Convert tpch .slt files to .parquet or .csv files +/// Prepare TPCH parquet files for benchmarks #[derive(Debug, StructOpt)] -pub struct ConvertOpt { - /// Path to csv files +pub struct PrepareTpchOpt { + /// Path to unprepared files #[structopt(parse(from_os_str), required = true, short = "i", long = "input")] input_path: PathBuf, @@ -40,38 +37,24 @@ pub struct ConvertOpt { #[structopt(parse(from_os_str), required = true, short = "o", long = "output")] output_path: PathBuf, - /// Output file format: `csv` or `parquet` - #[structopt(short = "f", long = "format")] - file_format: String, - - /// Compression to use when writing Parquet files - #[structopt(short = "c", long = "compression", default_value = "zstd")] - compression: String, - /// Number of partitions to produce. By default, uses only 1 partition. #[structopt(short = "n", long = "partitions", default_value = "1")] partitions: usize, - /// Batch size when reading CSV or Parquet files - #[structopt(short = "s", long = "batch-size", default_value = "8192")] - batch_size: usize, - /// Sort each table by its first column in ascending order. #[structopt(short = "t", long = "sort")] sort: bool, } -impl ConvertOpt { +impl PrepareTpchOpt { pub async fn run(self) -> Result<()> { - let compression = self.compression()?; - let input_path = self.input_path.to_str().unwrap(); let output_path = self.output_path.to_str().unwrap(); let output_root_path = Path::new(output_path); - for table in TPCH_TABLES { + for table in tpch::TPCH_TABLES { let start = Instant::now(); - let schema = get_tbl_tpch_table_schema(table); + let schema = tpch::get_tpch_table_schema(table); let key_column_name = schema.fields()[0].name(); let input_path = format!("{input_path}/{table}.tbl"); @@ -87,9 +70,7 @@ impl ConvertOpt { options }; - let config = SessionConfig::new() - .with_target_partitions(self.partitions) - .with_batch_size(self.batch_size); + let config = SessionConfig::new().with_target_partitions(self.partitions); let ctx = SessionContext::new_with_config(config); // build plan to read the TBL file @@ -119,40 +100,16 @@ impl ConvertOpt { let output_path = output_path.to_str().unwrap().to_owned(); fs::create_dir_all(&output_path)?; println!( - "Converting '{}' to {} files in directory '{}'", - &input_path, self.file_format, &output_path + "Converting '{}' to parquet files in directory '{}'", + &input_path, &output_path ); - match self.file_format.as_str() { - "csv" => ctx.write_csv(csv, output_path).await?, - "parquet" => { - let props = WriterProperties::builder() - .set_compression(compression) - .build(); - ctx.write_parquet(csv, output_path, Some(props)).await? - } - other => { - return not_impl_err!("Invalid output format: {other}"); - } - } + let props = WriterProperties::builder() + .set_compression(Compression::ZSTD(Default::default())) + .build(); + ctx.write_parquet(csv, output_path, Some(props)).await?; println!("Conversion completed in {} ms", start.elapsed().as_millis()); } Ok(()) } - - /// return the compression method to use when writing parquet - fn compression(&self) -> Result { - Ok(match self.compression.as_str() { - "none" => Compression::UNCOMPRESSED, - "snappy" => Compression::SNAPPY, - "brotli" => Compression::BROTLI(Default::default()), - "gzip" => Compression::GZIP(Default::default()), - "lz4" => Compression::LZ4, - "lz0" => Compression::LZO, - "zstd" => Compression::ZSTD(Default::default()), - other => { - return not_impl_err!("Invalid compression format: {other}"); - } - }) - } } diff --git a/benchmarks/src/run.rs b/benchmarks/src/run.rs new file mode 100644 index 00000000..03300133 --- /dev/null +++ b/benchmarks/src/run.rs @@ -0,0 +1,621 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use chrono::{DateTime, Utc}; +use datafusion::DATAFUSION_VERSION; +use datafusion::arrow::record_batch::RecordBatch; +use datafusion::arrow::util::pretty::pretty_format_batches; +use datafusion::common::instant::Instant; +use datafusion::common::tree_node::{Transformed, TreeNode}; +use datafusion::common::utils::get_available_parallelism; +use datafusion::common::{exec_err, not_impl_err}; +use datafusion::error::{DataFusionError, Result}; +use datafusion::execution::SessionStateBuilder; +use datafusion::physical_plan::display::DisplayableExecutionPlan; +use datafusion::physical_plan::{collect, displayable}; +use datafusion::prelude::*; +use datafusion_distributed::test_utils::localhost::LocalHostWorkerResolver; +use datafusion_distributed::test_utils::{tpcds, tpch}; +use datafusion_distributed::{ + ArrowFlightEndpoint, DistributedExt, DistributedPhysicalOptimizerRule, NetworkBoundaryExt, +}; +use log::info; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::error::Error; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Duration, SystemTime}; +use structopt::StructOpt; +use tokio::net::TcpListener; +use tonic::codegen::tokio_stream; +use tonic::transport::Server; + +/// Run the tpch benchmark. +/// +/// This benchmarks is derived from the [TPC-H][1] version +/// [2.17.1]. The data and answers are generated using `tpch-gen` from +/// [2]. +/// +/// [1]: http://www.tpc.org/tpch/ +/// [2]: https://github.com/databricks/tpch-dbgen.git +/// [2.17.1]: https://www.tpc.org/tpc_documents_current_versions/pdf/tpc-h_v2.17.1.pdf +#[derive(Debug, StructOpt, Clone)] +#[structopt(verbatim_doc_comment)] +pub struct RunOpt { + /// Query number. If not specified, runs all queries + #[structopt(short, long)] + pub query: Option, + + /// Path to data files + #[structopt(parse(from_os_str), short = "p", long = "path")] + path: Option, + + /// Path to machine readable output file + #[structopt(parse(from_os_str), short = "o", long = "output")] + output_path: Option, + + /// Spawns a worker in the specified port. + #[structopt(long)] + spawn: Option, + + /// The ports of all the workers involved in the query. + #[structopt(long, use_delimiter = true)] + workers: Vec, + + /// Number of physical threads per worker. + #[structopt(long)] + threads: Option, + + /// Number of files per each distributed task. + #[structopt(long)] + files_per_task: Option, + + /// Task count scale factor for when nodes in stages change the cardinality of the data + #[structopt(long)] + cardinality_task_sf: Option, + + /// Collects metrics across network boundaries + #[structopt(long)] + collect_metrics: bool, + + /// Number of iterations of each test run + #[structopt(short = "i", long = "iterations", default_value = "3")] + iterations: usize, + + /// Number of partitions to process in parallel. Defaults to number of available cores. + /// Should typically be less or equal than --threads. + #[structopt(short = "n", long = "partitions")] + partitions: Option, + + /// Batch size when reading CSV or Parquet files + #[structopt(short = "s", long = "batch-size")] + batch_size: Option, + + /// Activate debug mode to see more details + #[structopt(short, long)] + debug: bool, +} + +#[derive(Debug)] +enum Dataset { + Tpch, + Tpcds, +} + +impl Dataset { + fn infer_from_data_path(path: PathBuf) -> Result { + if path + .iter() + .any(|v| v.to_str().is_some_and(|v| v.contains("tpch"))) + { + return Ok(Self::Tpch); + } + if path + .iter() + .any(|v| v.to_str().is_some_and(|v| v.contains("tpcds"))) + { + return Ok(Self::Tpcds); + } + not_impl_err!( + "Cannot infer benchmark dataset from path {}", + path.display() + ) + } + + fn queries(&self) -> Result, DataFusionError> { + match self { + Dataset::Tpch => (1..22 + 1) + .map(|i| Ok((i as usize, tpch::get_test_tpch_query(i)?))) + .collect(), + Dataset::Tpcds => (1..99 + 1) + .map(|i| Ok((i, tpcds::get_test_tpcds_query(i)?))) + .collect(), + } + } +} + +impl RunOpt { + fn config(&self) -> Result { + SessionConfig::from_env().map(|mut config| { + if let Some(batch_size) = self.batch_size { + config = config.with_batch_size(batch_size); + } + if let Some(partitions) = self.partitions { + config = config.with_target_partitions(partitions); + } + config + }) + } + + pub fn run(self) -> Result<()> { + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(self.threads.unwrap_or(get_available_parallelism())) + .enable_all() + .build()?; + + if let Some(port) = self.spawn { + rt.block_on(async move { + let listener = TcpListener::bind(format!("127.0.0.1:{port}")).await?; + println!("Listening on {}...", listener.local_addr().unwrap()); + let incoming = tokio_stream::wrappers::TcpListenerStream::new(listener); + Ok::<_, Box>( + Server::builder() + .add_service(ArrowFlightEndpoint::default().into_flight_server()) + .serve_with_incoming(incoming) + .await?, + ) + })?; + } else { + rt.block_on(self.run_local())?; + } + Ok(()) + } + + async fn run_local(mut self) -> Result<()> { + let config = self.config()?.with_target_partitions(self.partitions()); + let state = SessionStateBuilder::new() + .with_default_features() + .with_config(config) + .with_distributed_worker_resolver(LocalHostWorkerResolver::new(self.workers.clone())) + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + .with_distributed_files_per_task( + self.files_per_task.unwrap_or(get_available_parallelism()), + )? + .with_distributed_cardinality_effect_task_scale_factor( + self.cardinality_task_sf.unwrap_or(1.0), + )? + .with_distributed_metrics_collection(self.collect_metrics)? + .build(); + let ctx = SessionContext::new_with_state(state); + let path = self.get_path()?; + self.register_tables(&ctx, path.clone()).await?; + + println!("Running benchmarks with the following options: {self:?}"); + self.output_path.get_or_insert(path.join("results.json")); + let mut benchmark_run = BenchmarkRun::new( + self.workers.len(), + self.threads.unwrap_or(get_available_parallelism()), + ); + + let dataset = Dataset::infer_from_data_path(path.clone())?; + + for (id, sql) in dataset.queries()? { + if self.query.is_some_and(|v| v != id) { + continue; + } + let query_id = format!("{dataset:?} {id}"); + benchmark_run.start_new_case(&query_id); + let query_run = self.benchmark_query(&query_id, &sql, &ctx).await; + match query_run { + Ok(query_results) => { + for iter in query_results { + benchmark_run.write_iter(iter); + } + } + Err(e) => { + benchmark_run.mark_failed(); + eprintln!("{query_id} failed: {e:?}"); + } + } + } + benchmark_run.maybe_compare_with_previous(self.output_path.as_ref())?; + benchmark_run.maybe_write_json(self.output_path.as_ref())?; + benchmark_run.maybe_print_failures(); + Ok(()) + } + + async fn benchmark_query( + &self, + id: &str, + sql: &str, + ctx: &SessionContext, + ) -> Result> { + let mut millis = vec![]; + // run benchmark + let mut query_results = vec![]; + + let mut n_tasks = 0; + for i in 0..self.iterations { + let start = Instant::now(); + let mut result = vec![]; + + for query in sql.split(";").map(|v| v.trim()) { + if query.starts_with("create") || query.starts_with("drop") { + self.execute_query(ctx, query).await?; + } else if !query.is_empty() { + (result, n_tasks) = self.execute_query(ctx, query).await?; + } + } + + let elapsed = start.elapsed(); + let ms = elapsed.as_secs_f64() * 1000.0; + millis.push(ms); + info!("output:\n\n{}\n\n", pretty_format_batches(&result)?); + let row_count = result.iter().map(|b| b.num_rows()).sum(); + println!("Query {id} iteration {i} took {ms:.1} ms and returned {row_count} rows"); + + query_results.push(QueryIter { + elapsed, + row_count, + n_tasks, + }); + } + + let avg = millis.iter().sum::() / millis.len() as f64; + println!("Query {id} avg time: {avg:.2} ms"); + if n_tasks > 0 { + println!("Query {id} number of tasks: {n_tasks}"); + } + + Ok(query_results) + } + + async fn register_tables(&self, ctx: &SessionContext, path: PathBuf) -> Result<()> { + for entry in fs::read_dir(path)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + let table_name = path.file_name().unwrap().to_str().unwrap(); + ctx.register_parquet( + table_name, + path.display().to_string(), + ParquetReadOptions::default(), + ) + .await?; + } + } + Ok(()) + } + + async fn execute_query( + &self, + ctx: &SessionContext, + sql: &str, + ) -> Result<(Vec, usize)> { + let plan = ctx.sql(sql).await?; + let (state, plan) = plan.into_parts(); + + if self.debug { + println!("=== Logical plan ===\n{plan}\n"); + } + + let plan = state.optimize(&plan)?; + if self.debug { + println!("=== Optimized logical plan ===\n{plan}\n"); + } + let physical_plan = state.create_physical_plan(&plan).await?; + if self.debug { + println!( + "=== Physical plan ===\n{}\n", + displayable(physical_plan.as_ref()).indent(true) + ); + } + let mut n_tasks = 0; + physical_plan.clone().transform_down(|node| { + if let Some(node) = node.as_network_boundary() { + n_tasks += node.input_stage().tasks.len() + } + Ok(Transformed::no(node)) + })?; + let result = collect(physical_plan.clone(), state.task_ctx()).await?; + if self.debug { + println!( + "=== Physical plan with metrics ===\n{}\n", + DisplayableExecutionPlan::with_metrics(physical_plan.as_ref()).indent(true) + ); + } + Ok((result, n_tasks)) + } + + fn get_path(&self) -> Result { + if let Some(path) = &self.path { + return Ok(path.clone()); + } + let crate_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let data_path = crate_path.join("data"); + let entries = fs::read_dir(&data_path)?.collect::, _>>()?; + if entries.is_empty() { + exec_err!( + "No Benchmarking dataset present in '{data_path:?}'. Generate one with ./benchmarks/gen-tpch.sh" + ) + } else if entries.len() == 1 { + Ok(entries[0].path()) + } else { + exec_err!( + "Multiple Benchmarking datasets present in '{data_path:?}'. One must be selected with --path" + ) + } + } + + fn partitions(&self) -> usize { + if let Some(partitions) = self.partitions { + return partitions; + } + if let Some(threads) = self.threads { + return threads; + } + get_available_parallelism() + } +} + +fn serialize_start_time(start_time: &SystemTime, ser: S) -> Result +where + S: Serializer, +{ + ser.serialize_u64( + start_time + .duration_since(SystemTime::UNIX_EPOCH) + .expect("current time is later than UNIX_EPOCH") + .as_secs(), + ) +} +fn deserialize_start_time<'de, D>(des: D) -> Result +where + D: Deserializer<'de>, +{ + let secs = u64::deserialize(des)?; + Ok(SystemTime::UNIX_EPOCH + Duration::from_secs(secs)) +} + +fn serialize_elapsed(elapsed: &Duration, ser: S) -> Result +where + S: Serializer, +{ + let ms = elapsed.as_secs_f64() * 1000.0; + ser.serialize_f64(ms) +} + +fn deserialize_elapsed<'de, D>(des: D) -> Result +where + D: Deserializer<'de>, +{ + let ms = f64::deserialize(des)?; + Ok(Duration::from_secs_f64(ms / 1000.0)) +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct RunContext { + /// Benchmark crate version + pub benchmark_version: String, + /// DataFusion crate version + pub datafusion_version: String, + /// Number of CPU cores + pub num_cpus: usize, + /// Number of workers involved in a distributed query + pub workers: usize, + /// Number of physical threads used per worker + pub threads: usize, + /// Start time + #[serde( + serialize_with = "serialize_start_time", + deserialize_with = "deserialize_start_time" + )] + pub start_time: SystemTime, + /// CLI arguments + pub arguments: Vec, +} + +impl RunContext { + pub fn new(workers: usize, threads: usize) -> Self { + Self { + benchmark_version: env!("CARGO_PKG_VERSION").to_owned(), + datafusion_version: DATAFUSION_VERSION.to_owned(), + num_cpus: get_available_parallelism(), + workers, + threads, + start_time: SystemTime::now(), + arguments: std::env::args().skip(1).collect::>(), + } + } +} + +/// A single iteration of a benchmark query +#[derive(Debug, Serialize, Deserialize)] +pub struct QueryIter { + #[serde( + serialize_with = "serialize_elapsed", + deserialize_with = "deserialize_elapsed" + )] + pub elapsed: Duration, + pub row_count: usize, + pub n_tasks: usize, +} +/// A single benchmark case +#[derive(Debug, Serialize, Deserialize)] +pub struct BenchQuery { + query: String, + iterations: Vec, + #[serde( + serialize_with = "serialize_start_time", + deserialize_with = "deserialize_start_time" + )] + start_time: SystemTime, + success: bool, +} + +/// collects benchmark run data and then serializes it at the end +#[derive(Debug, Serialize, Deserialize)] +pub struct BenchmarkRun { + context: RunContext, + queries: Vec, + current_case: Option, +} + +impl BenchmarkRun { + // create new + pub fn new(workers: usize, threads: usize) -> Self { + Self { + context: RunContext::new(workers, threads), + queries: vec![], + current_case: None, + } + } + /// begin a new case. iterations added after this will be included in the new case + pub fn start_new_case(&mut self, id: &str) { + self.queries.push(BenchQuery { + query: id.to_owned(), + iterations: vec![], + start_time: SystemTime::now(), + success: true, + }); + if let Some(c) = self.current_case.as_mut() { + *c += 1; + } else { + self.current_case = Some(0); + } + } + /// Write a new iteration to the current case + pub fn write_iter(&mut self, query_iter: QueryIter) { + if let Some(idx) = self.current_case { + self.queries[idx].iterations.push(query_iter) + } else { + panic!("no cases existed yet"); + } + } + + /// Print the names of failed queries, if any + pub fn maybe_print_failures(&self) { + let failed_queries: Vec<&str> = self + .queries + .iter() + .filter_map(|q| (!q.success).then_some(q.query.as_str())) + .collect(); + + if !failed_queries.is_empty() { + println!("Failed Queries: {}", failed_queries.join(", ")); + } + } + + /// Mark current query + pub fn mark_failed(&mut self) { + if let Some(idx) = self.current_case { + self.queries[idx].success = false; + } else { + unreachable!("Cannot mark failure: no current case"); + } + } + + /// Stringify data into formatted json + pub fn to_json(&self) -> String { + serde_json::to_string_pretty(&self).unwrap() + } + + /// Write data as json into output path if it exists. + pub fn maybe_write_json(&self, maybe_path: Option>) -> Result<()> { + if let Some(path) = maybe_path { + fs::write(path, self.to_json())?; + }; + Ok(()) + } + + pub fn maybe_compare_with_previous(&self, maybe_path: Option>) -> Result<()> { + let Some(path) = maybe_path else { + return Ok(()); + }; + let Ok(prev) = fs::read(path) else { + return Ok(()); + }; + + let Ok(prev_output) = serde_json::from_slice::(&prev) else { + return Ok(()); + }; + + let mut header_printed = false; + for query in self.queries.iter() { + let Some(prev_query) = prev_output.queries.iter().find(|v| v.query == query.query) + else { + continue; + }; + if prev_query.iterations.is_empty() { + continue; + } + if query.iterations.is_empty() { + println!("{}: Failed ❌", query.query); + continue; + } + + let avg_prev = prev_query.avg(); + let avg = query.avg(); + let (f, tag, emoji) = if avg < avg_prev { + let f = avg_prev as f64 / avg as f64; + (f, "faster", if f > 1.2 { "✅" } else { "✔" }) + } else { + let f = avg as f64 / avg_prev as f64; + (f, "slower", if f > 1.2 { "❌" } else { "✖" }) + }; + if !header_printed { + header_printed = true; + let datetime: DateTime = prev_query.start_time.into(); + let header = format!( + "==== Comparison with the previous benchmark from {} ====", + datetime.format("%Y-%m-%d %H:%M:%S UTC") + ); + println!("{header}"); + // Print machine information + println!("os: {}", std::env::consts::OS); + println!("arch: {}", std::env::consts::ARCH); + println!("cpu cores: {}", get_available_parallelism()); + println!( + "threads: {} -> {}", + prev_output.context.threads, self.context.threads + ); + println!( + "workers: {} -> {}", + prev_output.context.workers, self.context.workers + ); + println!("{}", "=".repeat(header.len())) + } + println!( + "{:>8}: prev={avg_prev:>4} ms, new={avg:>4} ms, diff={f:.2} {tag} {emoji}", + query.query + ); + } + + Ok(()) + } +} + +impl BenchQuery { + fn avg(&self) -> u128 { + self.iterations + .iter() + .map(|v| v.elapsed.as_millis()) + .sum::() + / self.iterations.len() as u128 + } +} diff --git a/benchmarks/src/tpch/mod.rs b/benchmarks/src/tpch/mod.rs deleted file mode 100644 index e8f558c9..00000000 --- a/benchmarks/src/tpch/mod.rs +++ /dev/null @@ -1,188 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Benchmark derived from TPC-H. This is not an official TPC-H benchmark. - -use datafusion::arrow::datatypes::SchemaBuilder; -use datafusion::{ - arrow::datatypes::{DataType, Field, Schema}, - common::plan_err, - error::Result, -}; -mod run; -pub use run::RunOpt; - -mod convert; -pub use convert::ConvertOpt; - -pub const TPCH_TABLES: &[&str] = &[ - "part", "supplier", "partsupp", "customer", "orders", "lineitem", "nation", "region", -]; - -pub const TPCH_QUERY_START_ID: usize = 1; -pub const TPCH_QUERY_END_ID: usize = 22; - -/// The `.tbl` file contains a trailing column -pub fn get_tbl_tpch_table_schema(table: &str) -> Schema { - let mut schema = SchemaBuilder::from(get_tpch_table_schema(table).fields); - schema.push(Field::new("__placeholder", DataType::Utf8, false)); - schema.finish() -} - -/// Get the schema for the benchmarks derived from TPC-H -pub fn get_tpch_table_schema(table: &str) -> Schema { - // note that the schema intentionally uses signed integers so that any generated Parquet - // files can also be used to benchmark tools that only support signed integers, such as - // Apache Spark - - match table { - "part" => Schema::new(vec![ - Field::new("p_partkey", DataType::Int64, false), - Field::new("p_name", DataType::Utf8, false), - Field::new("p_mfgr", DataType::Utf8, false), - Field::new("p_brand", DataType::Utf8, false), - Field::new("p_type", DataType::Utf8, false), - Field::new("p_size", DataType::Int32, false), - Field::new("p_container", DataType::Utf8, false), - Field::new("p_retailprice", DataType::Decimal128(15, 2), false), - Field::new("p_comment", DataType::Utf8, false), - ]), - - "supplier" => Schema::new(vec![ - Field::new("s_suppkey", DataType::Int64, false), - Field::new("s_name", DataType::Utf8, false), - Field::new("s_address", DataType::Utf8, false), - Field::new("s_nationkey", DataType::Int64, false), - Field::new("s_phone", DataType::Utf8, false), - Field::new("s_acctbal", DataType::Decimal128(15, 2), false), - Field::new("s_comment", DataType::Utf8, false), - ]), - - "partsupp" => Schema::new(vec![ - Field::new("ps_partkey", DataType::Int64, false), - Field::new("ps_suppkey", DataType::Int64, false), - Field::new("ps_availqty", DataType::Int32, false), - Field::new("ps_supplycost", DataType::Decimal128(15, 2), false), - Field::new("ps_comment", DataType::Utf8, false), - ]), - - "customer" => Schema::new(vec![ - Field::new("c_custkey", DataType::Int64, false), - Field::new("c_name", DataType::Utf8, false), - Field::new("c_address", DataType::Utf8, false), - Field::new("c_nationkey", DataType::Int64, false), - Field::new("c_phone", DataType::Utf8, false), - Field::new("c_acctbal", DataType::Decimal128(15, 2), false), - Field::new("c_mktsegment", DataType::Utf8, false), - Field::new("c_comment", DataType::Utf8, false), - ]), - - "orders" => Schema::new(vec![ - Field::new("o_orderkey", DataType::Int64, false), - Field::new("o_custkey", DataType::Int64, false), - Field::new("o_orderstatus", DataType::Utf8, false), - Field::new("o_totalprice", DataType::Decimal128(15, 2), false), - Field::new("o_orderdate", DataType::Date32, false), - Field::new("o_orderpriority", DataType::Utf8, false), - Field::new("o_clerk", DataType::Utf8, false), - Field::new("o_shippriority", DataType::Int32, false), - Field::new("o_comment", DataType::Utf8, false), - ]), - - "lineitem" => Schema::new(vec![ - Field::new("l_orderkey", DataType::Int64, false), - Field::new("l_partkey", DataType::Int64, false), - Field::new("l_suppkey", DataType::Int64, false), - Field::new("l_linenumber", DataType::Int32, false), - Field::new("l_quantity", DataType::Decimal128(15, 2), false), - Field::new("l_extendedprice", DataType::Decimal128(15, 2), false), - Field::new("l_discount", DataType::Decimal128(15, 2), false), - Field::new("l_tax", DataType::Decimal128(15, 2), false), - Field::new("l_returnflag", DataType::Utf8, false), - Field::new("l_linestatus", DataType::Utf8, false), - Field::new("l_shipdate", DataType::Date32, false), - Field::new("l_commitdate", DataType::Date32, false), - Field::new("l_receiptdate", DataType::Date32, false), - Field::new("l_shipinstruct", DataType::Utf8, false), - Field::new("l_shipmode", DataType::Utf8, false), - Field::new("l_comment", DataType::Utf8, false), - ]), - - "nation" => Schema::new(vec![ - Field::new("n_nationkey", DataType::Int64, false), - Field::new("n_name", DataType::Utf8, false), - Field::new("n_regionkey", DataType::Int64, false), - Field::new("n_comment", DataType::Utf8, false), - ]), - - "region" => Schema::new(vec![ - Field::new("r_regionkey", DataType::Int64, false), - Field::new("r_name", DataType::Utf8, false), - Field::new("r_comment", DataType::Utf8, false), - ]), - - _ => unimplemented!(), - } -} - -fn get_benchmark_queries_dir() -> std::path::PathBuf { - std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../testdata/tpch/queries") -} - -/// Get the SQL statements from the specified query file -pub fn get_query_sql(query: usize) -> Result> { - if query > 0 && query < 23 { - let queries_dir = get_benchmark_queries_dir(); - let contents = datafusion_distributed::test_utils::tpch::tpch_query_from_dir( - &queries_dir, - query as u8, - ); - Ok(contents - .split(';') - .map(|s| s.trim()) - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()) - .collect()) - } else { - plan_err!("invalid query. Expected value between 1 and 22") - } -} - -pub const QUERY_LIMIT: [Option; 22] = [ - None, - Some(100), - Some(10), - None, - None, - None, - None, - None, - None, - Some(20), - None, - None, - None, - None, - None, - None, - None, - Some(100), - None, - None, - Some(100), - None, -]; diff --git a/benchmarks/src/tpch/run.rs b/benchmarks/src/tpch/run.rs deleted file mode 100644 index 63a2680b..00000000 --- a/benchmarks/src/tpch/run.rs +++ /dev/null @@ -1,439 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::{ - TPCH_QUERY_END_ID, TPCH_QUERY_START_ID, TPCH_TABLES, get_query_sql, get_tbl_tpch_table_schema, - get_tpch_table_schema, -}; -use crate::util::{ - BenchmarkRun, CommonOpt, InMemoryCacheExecCodec, InMemoryDataSourceRule, QueryIter, - WarmingUpMarker, -}; -use async_trait::async_trait; -use datafusion::arrow::record_batch::RecordBatch; -use datafusion::arrow::util::pretty::pretty_format_batches; -use datafusion::common::instant::Instant; -use datafusion::common::tree_node::{Transformed, TreeNode}; -use datafusion::common::utils::get_available_parallelism; -use datafusion::common::{DEFAULT_CSV_EXTENSION, DEFAULT_PARQUET_EXTENSION, exec_err}; -use datafusion::datasource::TableProvider; -use datafusion::datasource::file_format::FileFormat; -use datafusion::datasource::file_format::csv::CsvFormat; -use datafusion::datasource::file_format::parquet::ParquetFormat; -use datafusion::datasource::listing::{ - ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl, -}; -use datafusion::error::{DataFusionError, Result}; -use datafusion::execution::{SessionState, SessionStateBuilder}; -use datafusion::physical_plan::display::DisplayableExecutionPlan; -use datafusion::physical_plan::{collect, displayable}; -use datafusion::prelude::*; -use datafusion_distributed::test_utils::localhost::{ - LocalHostWorkerResolver, spawn_flight_service, -}; -use datafusion_distributed::{ - DistributedExt, DistributedPhysicalOptimizerRule, DistributedSessionBuilder, - DistributedSessionBuilderContext, NetworkBoundaryExt, -}; -use log::info; -use std::fs; -use std::path::PathBuf; -use std::sync::Arc; -use structopt::StructOpt; -use tokio::net::TcpListener; - -/// Run the tpch benchmark. -/// -/// This benchmarks is derived from the [TPC-H][1] version -/// [2.17.1]. The data and answers are generated using `tpch-gen` from -/// [2]. -/// -/// [1]: http://www.tpc.org/tpch/ -/// [2]: https://github.com/databricks/tpch-dbgen.git -/// [2.17.1]: https://www.tpc.org/tpc_documents_current_versions/pdf/tpc-h_v2.17.1.pdf -#[derive(Debug, StructOpt, Clone)] -#[structopt(verbatim_doc_comment)] -pub struct RunOpt { - /// Query number. If not specified, runs all queries - #[structopt(short, long)] - pub query: Option, - - /// Common options - #[structopt(flatten)] - common: CommonOpt, - - /// Path to data files - #[structopt(parse(from_os_str), short = "p", long = "path")] - path: Option, - - /// File format: `csv` or `parquet` - #[structopt(short = "f", long = "format", default_value = "parquet")] - file_format: String, - - /// Load the data into a MemTable before executing the query - #[structopt(short = "m", long = "mem-table")] - mem_table: bool, - - /// Path to machine readable output file - #[structopt(parse(from_os_str), short = "o", long = "output")] - output_path: Option, - - /// Whether to disable collection of statistics (and cost based optimizations) or not. - #[structopt(short = "S", long = "disable-statistics")] - disable_statistics: bool, - - /// Mark the first column of each table as sorted in ascending order. - /// The tables should have been created with the `--sort` option for this to have any effect. - #[structopt(short = "t", long = "sorted")] - sorted: bool, - - /// Spawns a worker in the specified port. - #[structopt(long)] - spawn: Option, - - /// The ports of all the workers involved in the query. - #[structopt(long, use_delimiter = true)] - workers: Vec, - - /// Number of physical threads per worker. - #[structopt(long)] - threads: Option, - - /// Number of files per each distributed task. - #[structopt(long)] - files_per_task: Option, - - /// Task count scale factor for when nodes in stages change the cardinality of the data - #[structopt(long)] - cardinality_task_sf: Option, - - /// Collects metrics across network boundaries - #[structopt(long)] - collect_metrics: bool, -} - -#[async_trait] -impl DistributedSessionBuilder for RunOpt { - async fn build_session_state( - &self, - ctx: DistributedSessionBuilderContext, - ) -> Result { - let rt_builder = self.common.runtime_env_builder()?; - let config = self - .common - .config()? - .with_target_partitions(self.partitions()) - .with_collect_statistics(!self.disable_statistics); - let mut builder = SessionStateBuilder::new() - .with_runtime_env(rt_builder.build_arc()?) - .with_default_features() - .with_config(config) - .with_distributed_user_codec(InMemoryCacheExecCodec) - .with_distributed_worker_resolver(LocalHostWorkerResolver::new(self.workers.clone())) - .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) - .with_distributed_option_extension_from_headers::(&ctx.headers)? - .with_distributed_files_per_task( - self.files_per_task.unwrap_or(get_available_parallelism()), - )? - .with_distributed_cardinality_effect_task_scale_factor( - self.cardinality_task_sf.unwrap_or(1.0), - )? - .with_distributed_metrics_collection(self.collect_metrics)?; - - if self.mem_table { - builder = builder.with_physical_optimizer_rule(Arc::new(InMemoryDataSourceRule)); - } - - Ok(builder.build()) - } -} - -impl RunOpt { - pub fn run(self) -> Result<()> { - let rt = tokio::runtime::Builder::new_multi_thread() - .worker_threads(self.threads.unwrap_or(get_available_parallelism())) - .enable_all() - .build()?; - - if let Some(port) = self.spawn { - rt.block_on(async move { - let listener = TcpListener::bind(format!("127.0.0.1:{port}")).await?; - println!("Listening on {}...", listener.local_addr().unwrap()); - spawn_flight_service(self, listener).await - })?; - } else { - rt.block_on(self.run_local())?; - } - Ok(()) - } - - async fn run_local(mut self) -> Result<()> { - let mut state = self.build_session_state(Default::default()).await?; - if self.mem_table { - state = SessionStateBuilder::from(state) - .with_distributed_option_extension(WarmingUpMarker::warming_up())? - .build(); - } - let ctx = SessionContext::new_with_state(state); - self.register_tables(&ctx).await?; - - println!("Running benchmarks with the following options: {self:?}"); - let query_range = match self.query { - Some(query_id) => query_id..=query_id, - None => TPCH_QUERY_START_ID..=TPCH_QUERY_END_ID, - }; - - self.output_path - .get_or_insert(self.get_path()?.join("results.json")); - let mut benchmark_run = BenchmarkRun::new( - self.workers.len(), - self.threads.unwrap_or(get_available_parallelism()), - ); - - // Warmup the cache for the in-memory mode. - if self.mem_table { - for query_id in query_range.clone() { - // put the WarmingUpMarker in the context, otherwise, queries will fail as the - // InMemoryCacheExec node will think they should already be warmed up. - for query in get_query_sql(query_id)? { - self.execute_query(&ctx, &query).await?; - } - println!("Query {query_id} data loaded in memory"); - } - } - - for query_id in query_range { - benchmark_run.start_new_case(&format!("Query {query_id}")); - let query_run = self.benchmark_query(query_id, &ctx).await; - match query_run { - Ok(query_results) => { - for iter in query_results { - benchmark_run.write_iter(iter); - } - } - Err(e) => { - benchmark_run.mark_failed(); - eprintln!("Query {query_id} failed: {e:?}"); - } - } - } - benchmark_run.maybe_compare_with_previous(self.output_path.as_ref())?; - benchmark_run.maybe_write_json(self.output_path.as_ref())?; - benchmark_run.maybe_print_failures(); - Ok(()) - } - - async fn benchmark_query( - &self, - query_id: usize, - ctx: &SessionContext, - ) -> Result> { - let mut millis = vec![]; - // run benchmark - let mut query_results = vec![]; - - let sql = &get_query_sql(query_id)?; - - let mut n_tasks = 0; - for i in 0..self.iterations() { - let start = Instant::now(); - let mut result = vec![]; - - // query 15 is special, with 3 statements. the second statement is the one from which we - // want to capture the results - let result_stmt = if query_id == 15 { 1 } else { sql.len() - 1 }; - - for (i, query) in sql.iter().enumerate() { - if i == result_stmt { - (result, n_tasks) = self.execute_query(ctx, query).await?; - } else { - self.execute_query(ctx, query).await?; - } - } - - let elapsed = start.elapsed(); - let ms = elapsed.as_secs_f64() * 1000.0; - millis.push(ms); - info!("output:\n\n{}\n\n", pretty_format_batches(&result)?); - let row_count = result.iter().map(|b| b.num_rows()).sum(); - println!( - "Query {query_id} iteration {i} took {ms:.1} ms and returned {row_count} rows" - ); - - query_results.push(QueryIter { - elapsed, - row_count, - n_tasks, - }); - } - - let avg = millis.iter().sum::() / millis.len() as f64; - println!("Query {query_id} avg time: {avg:.2} ms"); - if n_tasks > 0 { - println!("Query {query_id} number of tasks: {n_tasks}"); - } - - Ok(query_results) - } - - async fn register_tables(&self, ctx: &SessionContext) -> Result<()> { - for table in TPCH_TABLES { - ctx.register_table(*table, self.get_table(ctx, table).await?)?; - } - Ok(()) - } - - async fn execute_query( - &self, - ctx: &SessionContext, - sql: &str, - ) -> Result<(Vec, usize)> { - let debug = self.common.debug; - let plan = ctx.sql(sql).await?; - let (state, plan) = plan.into_parts(); - - if debug { - println!("=== Logical plan ===\n{plan}\n"); - } - - let plan = state.optimize(&plan)?; - if debug { - println!("=== Optimized logical plan ===\n{plan}\n"); - } - let physical_plan = state.create_physical_plan(&plan).await?; - if debug { - println!( - "=== Physical plan ===\n{}\n", - displayable(physical_plan.as_ref()).indent(true) - ); - } - let mut n_tasks = 0; - physical_plan.clone().transform_down(|node| { - if let Some(node) = node.as_network_boundary() { - n_tasks += node.input_stage().tasks.len() - } - Ok(Transformed::no(node)) - })?; - let result = collect(physical_plan.clone(), state.task_ctx()).await?; - if debug { - println!( - "=== Physical plan with metrics ===\n{}\n", - DisplayableExecutionPlan::with_metrics(physical_plan.as_ref()).indent(true) - ); - } - Ok((result, n_tasks)) - } - - fn get_path(&self) -> Result { - if let Some(path) = &self.path { - return Ok(path.clone()); - } - let crate_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - let data_path = crate_path.join("data"); - let entries = fs::read_dir(&data_path)?.collect::, _>>()?; - if entries.is_empty() { - exec_err!( - "No TPCH dataset present in '{data_path:?}'. Generate one with ./benchmarks/gen-tpch.sh" - ) - } else if entries.len() == 1 { - Ok(entries[0].path()) - } else { - exec_err!( - "Multiple TPCH datasets present in '{data_path:?}'. One must be selected with --path" - ) - } - } - - async fn get_table(&self, ctx: &SessionContext, table: &str) -> Result> { - let path = self.get_path()?; - let path = path.to_str().unwrap(); - let table_format = self.file_format.as_str(); - let target_partitions = self.partitions(); - - // Obtain a snapshot of the SessionState - let state = ctx.state(); - let (format, path, extension): (Arc, String, &'static str) = - match table_format { - // dbgen creates .tbl ('|' delimited) files without header - "tbl" => { - let path = format!("{path}/{table}.tbl"); - - let format = CsvFormat::default() - .with_delimiter(b'|') - .with_has_header(false); - - (Arc::new(format), path, ".tbl") - } - "csv" => { - let path = format!("{path}/csv/{table}"); - let format = CsvFormat::default() - .with_delimiter(b',') - .with_has_header(true); - - (Arc::new(format), path, DEFAULT_CSV_EXTENSION) - } - "parquet" => { - let path = format!("{path}/{table}"); - let format = ParquetFormat::default() - .with_options(ctx.state().table_options().parquet.clone()); - - (Arc::new(format), path, DEFAULT_PARQUET_EXTENSION) - } - other => { - unimplemented!("Invalid file format '{}'", other); - } - }; - - let table_path = ListingTableUrl::parse(path)?; - let options = ListingOptions::new(format) - .with_file_extension(extension) - .with_target_partitions(target_partitions) - .with_collect_stat(state.config().collect_statistics()); - let schema = match table_format { - "parquet" => options.infer_schema(&state, &table_path).await?, - "tbl" => Arc::new(get_tbl_tpch_table_schema(table)), - "csv" => Arc::new(get_tpch_table_schema(table)), - _ => unreachable!(), - }; - let options = if self.sorted { - let key_column_name = schema.fields()[0].name(); - options.with_file_sort_order(vec![vec![col(key_column_name).sort(true, false)]]) - } else { - options - }; - - let config = ListingTableConfig::new(table_path) - .with_listing_options(options) - .with_schema(schema); - - Ok(Arc::new(ListingTable::try_new(config)?)) - } - - fn iterations(&self) -> usize { - self.common.iterations - } - - fn partitions(&self) -> usize { - if let Some(partitions) = self.common.partitions { - return partitions; - } - if let Some(threads) = self.threads { - return threads; - } - get_available_parallelism() - } -} diff --git a/benchmarks/src/util/memory.rs b/benchmarks/src/util/memory.rs deleted file mode 100644 index cfa7591d..00000000 --- a/benchmarks/src/util/memory.rs +++ /dev/null @@ -1,215 +0,0 @@ -use dashmap::{DashMap, Entry}; -use datafusion::arrow::record_batch::RecordBatch; -use datafusion::common::tree_node::{Transformed, TreeNode}; -use datafusion::common::{exec_err, extensions_options, plan_err}; -use datafusion::config::{ConfigExtension, ConfigOptions}; -use datafusion::error::DataFusionError; -use datafusion::execution::{SendableRecordBatchStream, TaskContext}; -use datafusion::physical_optimizer::PhysicalOptimizerRule; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, displayable, -}; -use datafusion_proto::physical_plan::PhysicalExtensionCodec; -use futures::{FutureExt, StreamExt}; -use prost::Message; -use std::any::Any; -use std::fmt::Formatter; -use std::sync::{Arc, LazyLock}; -use tokio::sync::OnceCell; - -type Key = (String, usize); -type Value = Arc>>; -static CACHE: LazyLock> = LazyLock::new(DashMap::default); - -/// Caches all the record batches in a global [CACHE] on the first run, and serves -/// them from the cache in any subsequent run. -#[derive(Debug, Clone)] -pub struct InMemoryCacheExec { - inner: Arc, -} - -extensions_options! { - /// Marker used by the [InMemoryCacheExec] that determines wether its fine - /// to load data from disk because we are warming up, or not. - /// - /// If this marker is not present during InMemoryCacheExec::execute(), and - /// the data was not loaded in-memory already, the query will fail. - pub struct WarmingUpMarker { - is_warming_up: bool, default = false - } -} - -impl ConfigExtension for WarmingUpMarker { - const PREFIX: &'static str = "in-memory-cache-exec"; -} - -impl WarmingUpMarker { - pub fn warming_up() -> Self { - Self { - is_warming_up: true, - } - } -} - -impl ExecutionPlan for InMemoryCacheExec { - fn name(&self) -> &str { - "InMemoryDataSourceExec" - } - - fn as_any(&self) -> &dyn Any { - self - } - - fn properties(&self) -> &PlanProperties { - self.inner.properties() - } - - fn children(&self) -> Vec<&Arc> { - vec![&self.inner] - } - - fn with_new_children( - self: Arc, - children: Vec>, - ) -> datafusion::common::Result> { - Ok(Arc::new(Self { - inner: children[0].clone(), - })) - } - - fn execute( - &self, - partition: usize, - context: Arc, - ) -> datafusion::common::Result { - let once = { - let inner_display = displayable(self.inner.as_ref()).one_line().to_string(); - let entry = CACHE.entry((inner_display, partition)); - if matches!(entry, Entry::Vacant(_)) - && !context - .session_config() - .options() - .extensions - .get::() - .map(|v| v.is_warming_up) - .unwrap_or_default() - { - return exec_err!("InMemoryCacheExec is not yet warmed up"); - } - let once = entry.or_insert(Arc::new(OnceCell::new())); - once.value().clone() - }; - - let inner = Arc::clone(&self.inner); - - let stream = async move { - let batches = once - .get_or_try_init(|| async move { - let mut stream = inner.execute(partition, context)?; - let mut batches = vec![]; - while let Some(batch) = stream.next().await { - batches.push(batch?); - } - Ok::<_, DataFusionError>(batches) - }) - .await?; - Ok(batches.clone()) - } - .into_stream() - .map(|v| match v { - Ok(batch) => futures::stream::iter(batch.into_iter().map(Ok)).boxed(), - Err(err) => futures::stream::once(async { Err(err) }).boxed(), - }) - .flatten(); - - Ok(Box::pin(RecordBatchStreamAdapter::new( - self.inner.schema(), - stream, - ))) - } -} - -impl DisplayAs for InMemoryCacheExec { - fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { - writeln!(f, "InMemoryDataSourceExec") - } -} - -#[derive(Clone, PartialEq, ::prost::Message)] -struct InMemoryCacheExecProto { - #[prost(string, tag = "1")] - name: String, -} - -#[derive(Debug)] -pub struct InMemoryCacheExecCodec; - -impl PhysicalExtensionCodec for InMemoryCacheExecCodec { - fn try_decode( - &self, - buf: &[u8], - inputs: &[Arc], - _ctx: &TaskContext, - ) -> datafusion::common::Result> { - let Ok(proto) = InMemoryCacheExecProto::decode(buf) else { - return plan_err!("no InMemoryDataSourceExecProto"); - }; - if proto.name != "InMemoryDataSourceExec" { - return plan_err!("unsupported InMemoryDataSourceExec proto: {:?}", proto.name); - }; - Ok(Arc::new(InMemoryCacheExec { - inner: inputs[0].clone(), - })) - } - - fn try_encode( - &self, - node: Arc, - buf: &mut Vec, - ) -> datafusion::common::Result<()> { - if !node.as_any().is::() { - return plan_err!("no InMemoryDataSourceExec"); - }; - let proto = InMemoryCacheExecProto { - name: "InMemoryDataSourceExec".to_string(), - }; - let Ok(_) = proto.encode(buf) else { - return plan_err!("no InMemoryDataSourceExecProto"); - }; - - Ok(()) - } -} - -/// Wraps any plan without children with an [InMemoryCacheExec] node. -#[derive(Debug)] -pub struct InMemoryDataSourceRule; - -impl PhysicalOptimizerRule for InMemoryDataSourceRule { - fn optimize( - &self, - plan: Arc, - _config: &ConfigOptions, - ) -> datafusion::common::Result> { - Ok(plan - .transform_up(|plan| { - if plan.children().is_empty() { - Ok(Transformed::yes(Arc::new(InMemoryCacheExec { - inner: plan.clone(), - }))) - } else { - Ok(Transformed::no(plan)) - } - })? - .data) - } - - fn name(&self) -> &str { - "InMemoryDataSourceRule" - } - - fn schema_check(&self) -> bool { - true - } -} diff --git a/benchmarks/src/util/mod.rs b/benchmarks/src/util/mod.rs deleted file mode 100644 index 40d8d956..00000000 --- a/benchmarks/src/util/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Shared benchmark utilities -mod memory; -mod options; -mod run; - -pub use memory::{InMemoryCacheExecCodec, InMemoryDataSourceRule, WarmingUpMarker}; -pub use options::CommonOpt; -pub use run::{BenchmarkRun, QueryIter}; diff --git a/benchmarks/src/util/options.rs b/benchmarks/src/util/options.rs deleted file mode 100644 index 0d838685..00000000 --- a/benchmarks/src/util/options.rs +++ /dev/null @@ -1,154 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::{num::NonZeroUsize, sync::Arc}; - -use datafusion::common::{DataFusionError, Result}; -use datafusion::{ - execution::{ - disk_manager::DiskManagerBuilder, - memory_pool::{FairSpillPool, GreedyMemoryPool, MemoryPool, TrackConsumersPool}, - runtime_env::RuntimeEnvBuilder, - }, - prelude::SessionConfig, -}; -use structopt::StructOpt; - -// Common benchmark options (don't use doc comments otherwise this doc -// shows up in help files) -#[derive(Debug, StructOpt, Clone)] -pub struct CommonOpt { - /// Number of iterations of each test run - #[structopt(short = "i", long = "iterations", default_value = "3")] - pub iterations: usize, - - /// Number of partitions to process in parallel. Defaults to number of available cores. - /// Should typically be less or equal than --threads. - #[structopt(short = "n", long = "partitions")] - pub partitions: Option, - - /// Batch size when reading CSV or Parquet files - #[structopt(short = "s", long = "batch-size")] - pub batch_size: Option, - - /// The memory pool type to use, should be one of "fair" or "greedy" - #[structopt(long = "mem-pool-type", default_value = "fair")] - pub mem_pool_type: String, - - /// Memory limit (e.g. '100M', '1.5G'). If not specified, run all pre-defined memory limits for given query - /// if there's any, otherwise run with no memory limit. - #[structopt(long = "memory-limit", parse(try_from_str = parse_memory_limit))] - pub memory_limit: Option, - - /// The amount of memory to reserve for sort spill operations. DataFusion's default value will be used - /// if not specified. - #[structopt(long = "sort-spill-reservation-bytes", parse(try_from_str = parse_memory_limit))] - pub sort_spill_reservation_bytes: Option, - - /// Activate debug mode to see more details - #[structopt(short, long)] - pub debug: bool, -} - -impl CommonOpt { - /// Return an appropriately configured `SessionConfig` - pub fn config(&self) -> Result { - SessionConfig::from_env().map(|config| self.update_config(config)) - } - - /// Modify the existing config appropriately - pub fn update_config(&self, mut config: SessionConfig) -> SessionConfig { - if let Some(batch_size) = self.batch_size { - config = config.with_batch_size(batch_size); - } - - if let Some(partitions) = self.partitions { - config = config.with_target_partitions(partitions); - } - - if let Some(sort_spill_reservation_bytes) = self.sort_spill_reservation_bytes { - config = config.with_sort_spill_reservation_bytes(sort_spill_reservation_bytes); - } - - config - } - - /// Return an appropriately configured `RuntimeEnvBuilder` - pub fn runtime_env_builder(&self) -> Result { - let mut rt_builder = RuntimeEnvBuilder::new(); - const NUM_TRACKED_CONSUMERS: usize = 5; - if let Some(memory_limit) = self.memory_limit { - let pool: Arc = match self.mem_pool_type.as_str() { - "fair" => Arc::new(TrackConsumersPool::new( - FairSpillPool::new(memory_limit), - NonZeroUsize::new(NUM_TRACKED_CONSUMERS).unwrap(), - )), - "greedy" => Arc::new(TrackConsumersPool::new( - GreedyMemoryPool::new(memory_limit), - NonZeroUsize::new(NUM_TRACKED_CONSUMERS).unwrap(), - )), - _ => { - return Err(DataFusionError::Configuration(format!( - "Invalid memory pool type: {}", - self.mem_pool_type - ))); - } - }; - rt_builder = rt_builder - .with_memory_pool(pool) - .with_disk_manager_builder(DiskManagerBuilder::default()); - } - Ok(rt_builder) - } -} - -/// Parse memory limit from string to number of bytes -/// e.g. '1.5G', '100M' -> 1572864 -fn parse_memory_limit(limit: &str) -> Result { - let (number, unit) = limit.split_at(limit.len() - 1); - let number: f64 = number - .parse() - .map_err(|_| format!("Failed to parse number from memory limit '{limit}'"))?; - - match unit { - "K" => Ok((number * 1024.0) as usize), - "M" => Ok((number * 1024.0 * 1024.0) as usize), - "G" => Ok((number * 1024.0 * 1024.0 * 1024.0) as usize), - _ => Err(format!( - "Unsupported unit '{unit}' in memory limit '{limit}'" - )), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_memory_limit_all() { - // Test valid inputs - assert_eq!(parse_memory_limit("100K").unwrap(), 102400); - assert_eq!(parse_memory_limit("1.5M").unwrap(), 1572864); - assert_eq!(parse_memory_limit("2G").unwrap(), 2147483648); - - // Test invalid unit - assert!(parse_memory_limit("500X").is_err()); - - // Test invalid number - assert!(parse_memory_limit("abcM").is_err()); - } -} diff --git a/benchmarks/src/util/run.rs b/benchmarks/src/util/run.rs deleted file mode 100644 index 48569e8c..00000000 --- a/benchmarks/src/util/run.rs +++ /dev/null @@ -1,270 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use chrono::{DateTime, Utc}; -use datafusion::common::utils::get_available_parallelism; -use datafusion::{DATAFUSION_VERSION, error::Result}; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use std::{ - path::Path, - time::{Duration, SystemTime}, -}; - -fn serialize_start_time(start_time: &SystemTime, ser: S) -> Result -where - S: Serializer, -{ - ser.serialize_u64( - start_time - .duration_since(SystemTime::UNIX_EPOCH) - .expect("current time is later than UNIX_EPOCH") - .as_secs(), - ) -} -fn deserialize_start_time<'de, D>(des: D) -> Result -where - D: Deserializer<'de>, -{ - let secs = u64::deserialize(des)?; - Ok(SystemTime::UNIX_EPOCH + Duration::from_secs(secs)) -} - -fn serialize_elapsed(elapsed: &Duration, ser: S) -> Result -where - S: Serializer, -{ - let ms = elapsed.as_secs_f64() * 1000.0; - ser.serialize_f64(ms) -} -fn deserialize_elapsed<'de, D>(des: D) -> Result -where - D: Deserializer<'de>, -{ - let ms = f64::deserialize(des)?; - Ok(Duration::from_secs_f64(ms / 1000.0)) -} -#[derive(Debug, Serialize, Deserialize)] -pub struct RunContext { - /// Benchmark crate version - pub benchmark_version: String, - /// DataFusion crate version - pub datafusion_version: String, - /// Number of CPU cores - pub num_cpus: usize, - /// Number of workers involved in a distributed query - pub workers: usize, - /// Number of physical threads used per worker - pub threads: usize, - /// Start time - #[serde( - serialize_with = "serialize_start_time", - deserialize_with = "deserialize_start_time" - )] - pub start_time: SystemTime, - /// CLI arguments - pub arguments: Vec, -} - -impl RunContext { - pub fn new(workers: usize, threads: usize) -> Self { - Self { - benchmark_version: env!("CARGO_PKG_VERSION").to_owned(), - datafusion_version: DATAFUSION_VERSION.to_owned(), - num_cpus: get_available_parallelism(), - workers, - threads, - start_time: SystemTime::now(), - arguments: std::env::args().skip(1).collect::>(), - } - } -} - -/// A single iteration of a benchmark query -#[derive(Debug, Serialize, Deserialize)] -pub struct QueryIter { - #[serde( - serialize_with = "serialize_elapsed", - deserialize_with = "deserialize_elapsed" - )] - pub elapsed: Duration, - pub row_count: usize, - pub n_tasks: usize, -} -/// A single benchmark case -#[derive(Debug, Serialize, Deserialize)] -pub struct BenchQuery { - query: String, - iterations: Vec, - #[serde( - serialize_with = "serialize_start_time", - deserialize_with = "deserialize_start_time" - )] - start_time: SystemTime, - success: bool, -} - -/// collects benchmark run data and then serializes it at the end -#[derive(Debug, Serialize, Deserialize)] -pub struct BenchmarkRun { - context: RunContext, - queries: Vec, - current_case: Option, -} - -impl BenchmarkRun { - // create new - pub fn new(workers: usize, threads: usize) -> Self { - Self { - context: RunContext::new(workers, threads), - queries: vec![], - current_case: None, - } - } - /// begin a new case. iterations added after this will be included in the new case - pub fn start_new_case(&mut self, id: &str) { - self.queries.push(BenchQuery { - query: id.to_owned(), - iterations: vec![], - start_time: SystemTime::now(), - success: true, - }); - if let Some(c) = self.current_case.as_mut() { - *c += 1; - } else { - self.current_case = Some(0); - } - } - /// Write a new iteration to the current case - pub fn write_iter(&mut self, query_iter: QueryIter) { - if let Some(idx) = self.current_case { - self.queries[idx].iterations.push(query_iter) - } else { - panic!("no cases existed yet"); - } - } - - /// Print the names of failed queries, if any - pub fn maybe_print_failures(&self) { - let failed_queries: Vec<&str> = self - .queries - .iter() - .filter_map(|q| (!q.success).then_some(q.query.as_str())) - .collect(); - - if !failed_queries.is_empty() { - println!("Failed Queries: {}", failed_queries.join(", ")); - } - } - - /// Mark current query - pub fn mark_failed(&mut self) { - if let Some(idx) = self.current_case { - self.queries[idx].success = false; - } else { - unreachable!("Cannot mark failure: no current case"); - } - } - - /// Stringify data into formatted json - pub fn to_json(&self) -> String { - serde_json::to_string_pretty(&self).unwrap() - } - - /// Write data as json into output path if it exists. - pub fn maybe_write_json(&self, maybe_path: Option>) -> Result<()> { - if let Some(path) = maybe_path { - std::fs::write(path, self.to_json())?; - }; - Ok(()) - } - - pub fn maybe_compare_with_previous(&self, maybe_path: Option>) -> Result<()> { - let Some(path) = maybe_path else { - return Ok(()); - }; - let Ok(prev) = std::fs::read(path) else { - return Ok(()); - }; - - let Ok(prev_output) = serde_json::from_slice::(&prev) else { - return Ok(()); - }; - - let mut header_printed = false; - for query in self.queries.iter() { - let Some(prev_query) = prev_output.queries.iter().find(|v| v.query == query.query) - else { - continue; - }; - if prev_query.iterations.is_empty() { - continue; - } - if query.iterations.is_empty() { - println!("{}: Failed ❌", query.query); - continue; - } - - let avg_prev = prev_query.avg(); - let avg = query.avg(); - let (f, tag, emoji) = if avg < avg_prev { - let f = avg_prev as f64 / avg as f64; - (f, "faster", if f > 1.2 { "✅" } else { "✔" }) - } else { - let f = avg as f64 / avg_prev as f64; - (f, "slower", if f > 1.2 { "❌" } else { "✖" }) - }; - if !header_printed { - header_printed = true; - let datetime: DateTime = prev_query.start_time.into(); - let header = format!( - "==== Comparison with the previous benchmark from {} ====", - datetime.format("%Y-%m-%d %H:%M:%S UTC") - ); - println!("{header}"); - // Print machine information - println!("os: {}", std::env::consts::OS); - println!("arch: {}", std::env::consts::ARCH); - println!("cpu cores: {}", get_available_parallelism()); - println!( - "threads: {} -> {}", - prev_output.context.threads, self.context.threads - ); - println!( - "workers: {} -> {}", - prev_output.context.workers, self.context.workers - ); - println!("{}", "=".repeat(header.len())) - } - println!( - "{:>8}: prev={avg_prev:>4} ms, new={avg:>4} ms, diff={f:.2} {tag} {emoji}", - query.query - ); - } - - Ok(()) - } -} - -impl BenchQuery { - fn avg(&self) -> u128 { - self.iterations - .iter() - .map(|v| v.elapsed.as_millis()) - .sum::() - / self.iterations.len() as u128 - } -} diff --git a/src/test_utils/tpch.rs b/src/test_utils/tpch.rs index 0d63f0e4..ff112dfb 100644 --- a/src/test_utils/tpch.rs +++ b/src/test_utils/tpch.rs @@ -5,10 +5,12 @@ use datafusion::{ catalog::{MemTable, TableProvider}, }; -use std::fs; - use arrow::record_batch::RecordBatch; +use datafusion::common::not_impl_datafusion_err; +use datafusion::error::DataFusionError; use parquet::{arrow::arrow_writer::ArrowWriter, file::properties::WriterProperties}; +use std::fs; +use std::path::Path; use tpchgen::generators::{ CustomerGenerator, LineItemGenerator, NationGenerator, OrderGenerator, PartGenerator, PartSuppGenerator, RegionGenerator, SupplierGenerator, @@ -18,15 +20,20 @@ use tpchgen_arrow::{ SupplierArrow, }; -pub fn tpch_query_from_dir(queries_dir: &std::path::Path, num: u8) -> String { +pub fn get_test_tpch_query(num: u8) -> Result { + let queries_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/tpch/queries"); let query_path = queries_dir.join(format!("q{num}.sql")); fs::read_to_string(query_path) - .unwrap_or_else(|_| panic!("Failed to read TPCH query file: q{num}.sql")) - .trim() - .to_string() + .map(|v| v.trim().to_string()) + .map_err(|_| not_impl_datafusion_err!("Failed to read TPCH query file: q{num}.sql")) } + pub const NUM_QUERIES: u8 = 22; // number of queries in the TPCH benchmark numbered from 1 to 22 +pub const TPCH_TABLES: &[&str] = &[ + "part", "supplier", "partsupp", "customer", "orders", "lineitem", "nation", "region", +]; + pub fn tpch_table(name: &str) -> Arc { let schema = Arc::new(get_tpch_table_schema(name)); Arc::new(MemTable::try_new(schema, vec![]).unwrap()) diff --git a/tests/tpcds_correctness_test.rs b/tests/tpcds_correctness_test.rs index 879486a3..51fda085 100644 --- a/tests/tpcds_correctness_test.rs +++ b/tests/tpcds_correctness_test.rs @@ -29,37 +29,37 @@ mod tests { test_tpcds_query(1).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_2() -> Result<()> { test_tpcds_query(2).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_3() -> Result<()> { test_tpcds_query(3).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_4() -> Result<()> { test_tpcds_query(4).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_5() -> Result<()> { test_tpcds_query(5).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_6() -> Result<()> { test_tpcds_query(6).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_7() -> Result<()> { test_tpcds_query(7).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_8() -> Result<()> { test_tpcds_query(8).await } @@ -70,102 +70,102 @@ mod tests { test_tpcds_query(9).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_10() -> Result<()> { test_tpcds_query(10).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_11() -> Result<()> { test_tpcds_query(11).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_12() -> Result<()> { test_tpcds_query(12).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_13() -> Result<()> { test_tpcds_query(13).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_14() -> Result<()> { test_tpcds_query(14).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_15() -> Result<()> { test_tpcds_query(15).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_16() -> Result<()> { test_tpcds_query(16).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_17() -> Result<()> { test_tpcds_query(17).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_18() -> Result<()> { test_tpcds_query(18).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_19() -> Result<()> { test_tpcds_query(19).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_20() -> Result<()> { test_tpcds_query(20).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_21() -> Result<()> { test_tpcds_query(21).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_22() -> Result<()> { test_tpcds_query(22).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_23() -> Result<()> { test_tpcds_query(23).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_24() -> Result<()> { test_tpcds_query(24).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_25() -> Result<()> { test_tpcds_query(25).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_26() -> Result<()> { test_tpcds_query(26).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_27() -> Result<()> { test_tpcds_query(27).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_28() -> Result<()> { test_tpcds_query(28).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_29() -> Result<()> { test_tpcds_query(29).await } @@ -176,207 +176,207 @@ mod tests { test_tpcds_query(30).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_31() -> Result<()> { test_tpcds_query(31).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_32() -> Result<()> { test_tpcds_query(32).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_33() -> Result<()> { test_tpcds_query(33).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_34() -> Result<()> { test_tpcds_query(34).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_35() -> Result<()> { test_tpcds_query(35).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_36() -> Result<()> { test_tpcds_query(36).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_37() -> Result<()> { test_tpcds_query(37).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_38() -> Result<()> { test_tpcds_query(38).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_39() -> Result<()> { test_tpcds_query(39).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_40() -> Result<()> { test_tpcds_query(40).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_41() -> Result<()> { test_tpcds_query(41).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_42() -> Result<()> { test_tpcds_query(42).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_43() -> Result<()> { test_tpcds_query(43).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_44() -> Result<()> { test_tpcds_query(44).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_45() -> Result<()> { test_tpcds_query(45).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_46() -> Result<()> { test_tpcds_query(46).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_47() -> Result<()> { test_tpcds_query(47).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_48() -> Result<()> { test_tpcds_query(48).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_49() -> Result<()> { test_tpcds_query(49).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_50() -> Result<()> { test_tpcds_query(50).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_51() -> Result<()> { test_tpcds_query(51).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_52() -> Result<()> { test_tpcds_query(52).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_53() -> Result<()> { test_tpcds_query(53).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_54() -> Result<()> { test_tpcds_query(54).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_55() -> Result<()> { test_tpcds_query(55).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_56() -> Result<()> { test_tpcds_query(56).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_57() -> Result<()> { test_tpcds_query(57).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_58() -> Result<()> { test_tpcds_query(58).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_59() -> Result<()> { test_tpcds_query(59).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_60() -> Result<()> { test_tpcds_query(60).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_61() -> Result<()> { test_tpcds_query(61).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_62() -> Result<()> { test_tpcds_query(62).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_63() -> Result<()> { test_tpcds_query(63).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_64() -> Result<()> { test_tpcds_query(64).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_65() -> Result<()> { test_tpcds_query(65).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_66() -> Result<()> { test_tpcds_query(66).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_67() -> Result<()> { test_tpcds_query(67).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_68() -> Result<()> { test_tpcds_query(68).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_69() -> Result<()> { test_tpcds_query(69).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_70() -> Result<()> { test_tpcds_query(70).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_71() -> Result<()> { test_tpcds_query(71).await } @@ -390,132 +390,132 @@ mod tests { test_tpcds_query(72).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_73() -> Result<()> { test_tpcds_query(73).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_74() -> Result<()> { test_tpcds_query(74).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_75() -> Result<()> { test_tpcds_query(75).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_76() -> Result<()> { test_tpcds_query(76).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_77() -> Result<()> { test_tpcds_query(77).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_78() -> Result<()> { test_tpcds_query(78).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_79() -> Result<()> { test_tpcds_query(79).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_80() -> Result<()> { test_tpcds_query(80).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_81() -> Result<()> { test_tpcds_query(81).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_82() -> Result<()> { test_tpcds_query(82).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_83() -> Result<()> { test_tpcds_query(83).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_84() -> Result<()> { test_tpcds_query(84).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_85() -> Result<()> { test_tpcds_query(85).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_86() -> Result<()> { test_tpcds_query(86).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_87() -> Result<()> { test_tpcds_query(87).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_88() -> Result<()> { test_tpcds_query(88).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_89() -> Result<()> { test_tpcds_query(89).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_90() -> Result<()> { test_tpcds_query(90).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_91() -> Result<()> { test_tpcds_query(91).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_92() -> Result<()> { test_tpcds_query(92).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_93() -> Result<()> { test_tpcds_query(93).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_94() -> Result<()> { test_tpcds_query(94).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_95() -> Result<()> { test_tpcds_query(95).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_96() -> Result<()> { test_tpcds_query(96).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_97() -> Result<()> { test_tpcds_query(97).await } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_98() -> Result<()> { test_tpcds_query(98).await } diff --git a/tests/tpch_correctness_test.rs b/tests/tpch_correctness_test.rs index 10de0789..65e990ca 100644 --- a/tests/tpch_correctness_test.rs +++ b/tests/tpch_correctness_test.rs @@ -18,52 +18,52 @@ mod tests { #[tokio::test] async fn test_tpch_1() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(1)).await + test_tpch_query(tpch::get_test_tpch_query(1)?).await } #[tokio::test] async fn test_tpch_2() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(2)).await + test_tpch_query(tpch::get_test_tpch_query(2)?).await } #[tokio::test] async fn test_tpch_3() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(3)).await + test_tpch_query(tpch::get_test_tpch_query(3)?).await } #[tokio::test] async fn test_tpch_4() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(4)).await + test_tpch_query(tpch::get_test_tpch_query(4)?).await } #[tokio::test] async fn test_tpch_5() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(5)).await + test_tpch_query(tpch::get_test_tpch_query(5)?).await } #[tokio::test] async fn test_tpch_6() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(6)).await + test_tpch_query(tpch::get_test_tpch_query(6)?).await } #[tokio::test] async fn test_tpch_7() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(7)).await + test_tpch_query(tpch::get_test_tpch_query(7)?).await } #[tokio::test] async fn test_tpch_8() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(8)).await + test_tpch_query(tpch::get_test_tpch_query(8)?).await } #[tokio::test] async fn test_tpch_9() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(9)).await + test_tpch_query(tpch::get_test_tpch_query(9)?).await } #[tokio::test] async fn test_tpch_10() -> Result<(), Box> { - let sql = get_test_tpch_query(10); + let sql = tpch::get_test_tpch_query(10)?; // There is a chance that this query returns non-deterministic results if two entries // happen to have the exact same revenue. With small scales, this never happens, but with // bigger scales, this is very likely to happen. @@ -74,62 +74,62 @@ mod tests { #[tokio::test] async fn test_tpch_11() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(11)).await + test_tpch_query(tpch::get_test_tpch_query(11)?).await } #[tokio::test] async fn test_tpch_12() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(12)).await + test_tpch_query(tpch::get_test_tpch_query(12)?).await } #[tokio::test] async fn test_tpch_13() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(13)).await + test_tpch_query(tpch::get_test_tpch_query(13)?).await } #[tokio::test] async fn test_tpch_14() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(14)).await + test_tpch_query(tpch::get_test_tpch_query(14)?).await } #[tokio::test] async fn test_tpch_15() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(15)).await + test_tpch_query(tpch::get_test_tpch_query(15)?).await } #[tokio::test] async fn test_tpch_16() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(16)).await + test_tpch_query(tpch::get_test_tpch_query(16)?).await } #[tokio::test] async fn test_tpch_17() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(17)).await + test_tpch_query(tpch::get_test_tpch_query(17)?).await } #[tokio::test] async fn test_tpch_18() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(18)).await + test_tpch_query(tpch::get_test_tpch_query(18)?).await } #[tokio::test] async fn test_tpch_19() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(19)).await + test_tpch_query(tpch::get_test_tpch_query(19)?).await } #[tokio::test] async fn test_tpch_20() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(20)).await + test_tpch_query(tpch::get_test_tpch_query(20)?).await } #[tokio::test] async fn test_tpch_21() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(21)).await + test_tpch_query(tpch::get_test_tpch_query(21)?).await } #[tokio::test] async fn test_tpch_22() -> Result<(), Box> { - test_tpch_query(get_test_tpch_query(22)).await + test_tpch_query(tpch::get_test_tpch_query(22)?).await } async fn test_tpch_query(sql: String) -> Result<(), Box> { @@ -197,11 +197,6 @@ mod tests { Ok(arrow::util::pretty::pretty_format_batches(&batches)?) } - pub fn get_test_tpch_query(num: u8) -> String { - let queries_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/tpch/queries"); - tpch::tpch_query_from_dir(&queries_dir, num) - } - // OnceCell to ensure TPCH tables are generated only once for tests static INIT_TEST_TPCH_TABLES: OnceCell<()> = OnceCell::const_new(); diff --git a/tests/tpch_explain_analyze.rs b/tests/tpch_explain_analyze.rs index fc8f2888..f2175767 100644 --- a/tests/tpch_explain_analyze.rs +++ b/tests/tpch_explain_analyze.rs @@ -1079,7 +1079,7 @@ mod tests { // and once in a non-distributed manner. For each query, it asserts that the results are identical. async fn run_tpch_query(ctx: SessionContext, query_id: u8) -> Result> { let data_dir = ensure_tpch_data(TPCH_SCALE_FACTOR, TPCH_DATA_PARTS).await; - let sql = get_test_tpch_query(query_id); + let sql = tpch::get_test_tpch_query(query_id)?; ctx.state_ref() .write() .config_mut() @@ -1126,11 +1126,6 @@ mod tests { Ok(explain_analyze(plan)?) } - pub fn get_test_tpch_query(num: u8) -> String { - let queries_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/tpch/queries"); - tpch::tpch_query_from_dir(&queries_dir, num) - } - // OnceCell to ensure TPCH tables are generated only once for tests static INIT_TEST_TPCH_TABLES: OnceCell<()> = OnceCell::const_new(); diff --git a/tests/tpch_plans_test.rs b/tests/tpch_plans_test.rs index 3dcee13d..f7e89163 100644 --- a/tests/tpch_plans_test.rs +++ b/tests/tpch_plans_test.rs @@ -1000,7 +1000,7 @@ mod tests { // and once in a non-distributed manner. For each query, it asserts that the results are identical. async fn run_tpch_query(ctx: SessionContext, query_id: u8) -> Result> { let data_dir = ensure_tpch_data(TPCH_SCALE_FACTOR, TPCH_DATA_PARTS).await; - let sql = get_test_tpch_query(query_id); + let sql = tpch::get_test_tpch_query(query_id)?; ctx.state_ref() .write() .config_mut() @@ -1044,11 +1044,6 @@ mod tests { Ok(display_plan_ascii(plan.as_ref(), false)) } - pub fn get_test_tpch_query(num: u8) -> String { - let queries_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/tpch/queries"); - tpch::tpch_query_from_dir(&queries_dir, num) - } - // OnceCell to ensure TPCH tables are generated only once for tests static INIT_TEST_TPCH_TABLES: OnceCell<()> = OnceCell::const_new(); From 831d61732b0f6d26ea2bc0d73b48a942c4e5532b Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Mon, 29 Dec 2025 13:03:15 +0100 Subject: [PATCH 06/27] Add Clickbench tests and benchmarks (#270) * Split channel resolver in two * Simplify WorkerResolverExtension and ChannelResolverExtension * Add default builder to ArrowFlightEndpoint * Add some docs * Listen to clippy * Split get_flight_client_for_url in two * Fix conflicts * Remove unnecessary channel resolver * Improve WorkerResolver docs * Use one ChannelResolver per runtime * Improve error reporting on client connection failure * Add a from_session_builder method for constructing an InMemoryChannelResolver * Add ChannelResolver and WorkerResolver default implementations for Arcs * Make TPC-DS tests use DataFusion test dataset * Remove non-working in-memory option from benchmarks * Remove unnecessary utils folder * Refactor benchmark folder * Rename to prepare_tpch.rs * Adapt benchmarks for TPC-DS * Update benchmarks README.md * Fix conflicts * Use default session state builder * Add clickbench correctness tests * Add clickbench plans tests * Ignore also clickbench 3 * Rename file * Add clickbench to benchmarks --- .github/workflows/ci.yml | 11 + .gitignore | 2 + Cargo.toml | 1 + benchmarks/gen-clickbench.sh | 22 + benchmarks/src/main.rs | 6 + benchmarks/src/prepare_clickbench.rs | 33 + benchmarks/src/run.rs | 34 +- src/test_utils/clickbench.rs | 106 +++ src/test_utils/mod.rs | 1 + testdata/clickbench/queries/q0.sql | 4 + testdata/clickbench/queries/q1.sql | 4 + testdata/clickbench/queries/q10.sql | 4 + testdata/clickbench/queries/q11.sql | 4 + testdata/clickbench/queries/q12.sql | 4 + testdata/clickbench/queries/q13.sql | 4 + testdata/clickbench/queries/q14.sql | 4 + testdata/clickbench/queries/q15.sql | 4 + testdata/clickbench/queries/q16.sql | 4 + testdata/clickbench/queries/q17.sql | 4 + testdata/clickbench/queries/q18.sql | 4 + testdata/clickbench/queries/q19.sql | 4 + testdata/clickbench/queries/q2.sql | 4 + testdata/clickbench/queries/q20.sql | 4 + testdata/clickbench/queries/q21.sql | 4 + testdata/clickbench/queries/q22.sql | 4 + testdata/clickbench/queries/q23.sql | 4 + testdata/clickbench/queries/q24.sql | 4 + testdata/clickbench/queries/q25.sql | 4 + testdata/clickbench/queries/q26.sql | 4 + testdata/clickbench/queries/q27.sql | 4 + testdata/clickbench/queries/q28.sql | 4 + testdata/clickbench/queries/q29.sql | 4 + testdata/clickbench/queries/q3.sql | 4 + testdata/clickbench/queries/q30.sql | 4 + testdata/clickbench/queries/q31.sql | 4 + testdata/clickbench/queries/q32.sql | 4 + testdata/clickbench/queries/q33.sql | 4 + testdata/clickbench/queries/q34.sql | 4 + testdata/clickbench/queries/q35.sql | 4 + testdata/clickbench/queries/q36.sql | 4 + testdata/clickbench/queries/q37.sql | 4 + testdata/clickbench/queries/q38.sql | 4 + testdata/clickbench/queries/q39.sql | 4 + testdata/clickbench/queries/q4.sql | 4 + testdata/clickbench/queries/q40.sql | 4 + testdata/clickbench/queries/q41.sql | 3 + testdata/clickbench/queries/q42.sql | 4 + testdata/clickbench/queries/q5.sql | 4 + testdata/clickbench/queries/q6.sql | 4 + testdata/clickbench/queries/q7.sql | 4 + testdata/clickbench/queries/q8.sql | 4 + testdata/clickbench/queries/q9.sql | 4 + tests/clickbench_correctness_test.rs | 316 +++++++++ tests/clickbench_plans_test.rs | 942 +++++++++++++++++++++++++++ 54 files changed, 1630 insertions(+), 15 deletions(-) create mode 100755 benchmarks/gen-clickbench.sh create mode 100644 benchmarks/src/prepare_clickbench.rs create mode 100644 src/test_utils/clickbench.rs create mode 100644 testdata/clickbench/queries/q0.sql create mode 100644 testdata/clickbench/queries/q1.sql create mode 100644 testdata/clickbench/queries/q10.sql create mode 100644 testdata/clickbench/queries/q11.sql create mode 100644 testdata/clickbench/queries/q12.sql create mode 100644 testdata/clickbench/queries/q13.sql create mode 100644 testdata/clickbench/queries/q14.sql create mode 100644 testdata/clickbench/queries/q15.sql create mode 100644 testdata/clickbench/queries/q16.sql create mode 100644 testdata/clickbench/queries/q17.sql create mode 100644 testdata/clickbench/queries/q18.sql create mode 100644 testdata/clickbench/queries/q19.sql create mode 100644 testdata/clickbench/queries/q2.sql create mode 100644 testdata/clickbench/queries/q20.sql create mode 100644 testdata/clickbench/queries/q21.sql create mode 100644 testdata/clickbench/queries/q22.sql create mode 100644 testdata/clickbench/queries/q23.sql create mode 100644 testdata/clickbench/queries/q24.sql create mode 100644 testdata/clickbench/queries/q25.sql create mode 100644 testdata/clickbench/queries/q26.sql create mode 100644 testdata/clickbench/queries/q27.sql create mode 100644 testdata/clickbench/queries/q28.sql create mode 100644 testdata/clickbench/queries/q29.sql create mode 100644 testdata/clickbench/queries/q3.sql create mode 100644 testdata/clickbench/queries/q30.sql create mode 100644 testdata/clickbench/queries/q31.sql create mode 100644 testdata/clickbench/queries/q32.sql create mode 100644 testdata/clickbench/queries/q33.sql create mode 100644 testdata/clickbench/queries/q34.sql create mode 100644 testdata/clickbench/queries/q35.sql create mode 100644 testdata/clickbench/queries/q36.sql create mode 100644 testdata/clickbench/queries/q37.sql create mode 100644 testdata/clickbench/queries/q38.sql create mode 100644 testdata/clickbench/queries/q39.sql create mode 100644 testdata/clickbench/queries/q4.sql create mode 100644 testdata/clickbench/queries/q40.sql create mode 100644 testdata/clickbench/queries/q41.sql create mode 100644 testdata/clickbench/queries/q42.sql create mode 100644 testdata/clickbench/queries/q5.sql create mode 100644 testdata/clickbench/queries/q6.sql create mode 100644 testdata/clickbench/queries/q7.sql create mode 100644 testdata/clickbench/queries/q8.sql create mode 100644 testdata/clickbench/queries/q9.sql create mode 100644 tests/clickbench_correctness_test.rs create mode 100644 tests/clickbench_plans_test.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca322307..b6a63a1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,17 @@ jobs: key: "main.zip" - run: cargo test --features tpcds --test 'tpcds_*' + clickbench-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup + - uses: actions/cache@v4 + with: + path: testdata/clickbench/ + key: "data" + - run: cargo test --features clickbench --test 'clickbench_*' + format-check: runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index 07512695..aa62b1fe 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ testdata/tpch/* testdata/tpcds/* !testdata/tpcds/queries !testdata/tpcds/README.md +testdata/clickbench/* +!testdata/clickbench/queries diff --git a/Cargo.toml b/Cargo.toml index 70f305b1..4770e155 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ integration = [ tpch = ["integration"] tpcds = ["integration"] +clickbench = ["integration"] [dev-dependencies] structopt = "0.3" diff --git a/benchmarks/gen-clickbench.sh b/benchmarks/gen-clickbench.sh new file mode 100755 index 00000000..c0b6d0af --- /dev/null +++ b/benchmarks/gen-clickbench.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -e + +PARTITION_START=${PARTITION_START:-0} +PARTITION_END=${PARTITION_END:-100} + +echo "Generating ClickBench dataset" + + +# https://stackoverflow.com/questions/59895/how-do-i-get-the-directory-where-a-bash-script-is-located-from-within-the-script +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +DATA_DIR=${DATA_DIR:-$SCRIPT_DIR/data} +CARGO_COMMAND=${CARGO_COMMAND:-"cargo run --release"} +CLICKBENCH_DIR="${DATA_DIR}/clickbench_${PARTITION_START}-${PARTITION_END}" + +echo "Creating clickbench dataset from partition ${PARTITION_START} to ${PARTITION_END}" + +# Ensure the target data directory exists +mkdir -p "${CLICKBENCH_DIR}" + +$CARGO_COMMAND -- prepare-clickbench --output "${CLICKBENCH_DIR}" --partition-start "$PARTITION_START" --partition-end "$PARTITION_END" diff --git a/benchmarks/src/main.rs b/benchmarks/src/main.rs index 1b60e1c6..2ccd262a 100644 --- a/benchmarks/src/main.rs +++ b/benchmarks/src/main.rs @@ -1,4 +1,5 @@ //! DataFusion Distributed benchmark runner +mod prepare_clickbench; mod prepare_tpcds; mod prepare_tpch; mod run; @@ -12,6 +13,7 @@ enum Options { Run(run::RunOpt), PrepareTpch(prepare_tpch::PrepareTpchOpt), PrepareTpcds(prepare_tpcds::PrepareTpcdsOpt), + PrepareClickbench(prepare_clickbench::PrepareClickBenchOpt), } // Main benchmark runner entrypoint @@ -28,5 +30,9 @@ pub fn main() -> Result<()> { let rt = tokio::runtime::Runtime::new()?; rt.block_on(async { opt.run().await }) } + Options::PrepareClickbench(opt) => { + let rt = tokio::runtime::Runtime::new()?; + rt.block_on(async { opt.run().await }) + } } } diff --git a/benchmarks/src/prepare_clickbench.rs b/benchmarks/src/prepare_clickbench.rs new file mode 100644 index 00000000..cb18c5bb --- /dev/null +++ b/benchmarks/src/prepare_clickbench.rs @@ -0,0 +1,33 @@ +use datafusion::error::DataFusionError; +use datafusion_distributed::test_utils::clickbench; +use std::path::{Path, PathBuf}; +use structopt::StructOpt; + +/// Prepare ClickBench parquet files for benchmarks +#[derive(Debug, StructOpt)] +pub struct PrepareClickBenchOpt { + /// Output path + #[structopt(parse(from_os_str), required = true, short = "o", long = "output")] + output_path: PathBuf, + + /// Clickbench dataset is partitioned in 100 files. You may not want to use all the files for + /// the benchmark, so this allows setting from which file partition to start. + #[structopt(long, default_value = "0")] + partition_start: usize, + + /// Clickbench dataset is partitioned in 100 files. You may not want to use all the files for + /// the benchmark, so this allows setting a maximum in the file partition index. + #[structopt(long, default_value = "100")] + partition_end: usize, +} + +impl PrepareClickBenchOpt { + pub async fn run(self) -> datafusion::common::Result<()> { + clickbench::generate_clickbench_data( + Path::new(&self.output_path), + self.partition_start..self.partition_end, + ) + .await + .map_err(|e| DataFusionError::Internal(format!("{e:?}"))) + } +} diff --git a/benchmarks/src/run.rs b/benchmarks/src/run.rs index 03300133..52900756 100644 --- a/benchmarks/src/run.rs +++ b/benchmarks/src/run.rs @@ -29,7 +29,7 @@ use datafusion::physical_plan::display::DisplayableExecutionPlan; use datafusion::physical_plan::{collect, displayable}; use datafusion::prelude::*; use datafusion_distributed::test_utils::localhost::LocalHostWorkerResolver; -use datafusion_distributed::test_utils::{tpcds, tpch}; +use datafusion_distributed::test_utils::{clickbench, tpcds, tpch}; use datafusion_distributed::{ ArrowFlightEndpoint, DistributedExt, DistributedPhysicalOptimizerRule, NetworkBoundaryExt, }; @@ -115,26 +115,27 @@ pub struct RunOpt { enum Dataset { Tpch, Tpcds, + Clickbench, } impl Dataset { fn infer_from_data_path(path: PathBuf) -> Result { - if path - .iter() - .any(|v| v.to_str().is_some_and(|v| v.contains("tpch"))) - { - return Ok(Self::Tpch); + fn path_contains(path: &Path, substr: &str) -> bool { + path.iter() + .any(|v| v.to_str().is_some_and(|v| v.contains(substr))) } - if path - .iter() - .any(|v| v.to_str().is_some_and(|v| v.contains("tpcds"))) - { - return Ok(Self::Tpcds); + if path_contains(&path, "tpch") { + Ok(Self::Tpch) + } else if path_contains(&path, "tpcds") { + Ok(Self::Tpcds) + } else if path_contains(&path, "clickbench") { + Ok(Self::Clickbench) + } else { + not_impl_err!( + "Cannot infer benchmark dataset from path {}", + path.display() + ) } - not_impl_err!( - "Cannot infer benchmark dataset from path {}", - path.display() - ) } fn queries(&self) -> Result, DataFusionError> { @@ -145,6 +146,9 @@ impl Dataset { Dataset::Tpcds => (1..99 + 1) .map(|i| Ok((i, tpcds::get_test_tpcds_query(i)?))) .collect(), + Dataset::Clickbench => (0..42 + 1) + .map(|i| Ok((i, clickbench::get_test_clickbench_query(i)?))) + .collect(), } } } diff --git a/src/test_utils/clickbench.rs b/src/test_utils/clickbench.rs new file mode 100644 index 00000000..fd3259c8 --- /dev/null +++ b/src/test_utils/clickbench.rs @@ -0,0 +1,106 @@ +use datafusion::common::{DataFusionError, internal_datafusion_err, internal_err}; +use datafusion::prelude::{ParquetReadOptions, SessionContext}; +use std::fs; +use std::io::Write; +use std::ops::Range; +use std::path::{Path, PathBuf}; +use tokio::task::JoinSet; + +const URL: &str = + "https://datasets.clickhouse.com/hits_compatible/athena_partitioned/hits_{}.parquet"; + +/// Load a single ClickBench query by ID (0-42). +pub fn get_test_clickbench_query(id: usize) -> Result { + let queries_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/clickbench/queries"); + + if !queries_dir.exists() { + return internal_err!( + "TPC-DS queries directory not found: {}", + queries_dir.display() + ); + } + + let query_file = queries_dir.join(format!("q{id}.sql")); + + if !query_file.exists() { + return internal_err!("Query file not found: {}", query_file.display()); + } + + let query_sql = fs::read_to_string(&query_file) + .map_err(|e| { + internal_datafusion_err!("Failed to read query file {}: {e}", query_file.display()) + })? + .trim() + .to_string(); + + Ok(query_sql) +} + +/// Downloads the datafusion-benchmarks repository as a zip file +async fn download_benchmark( + dest_path: PathBuf, + i: usize, +) -> Result<(), Box> { + if dest_path.exists() { + return Ok(()); + } + + // Create directory if it doesn't exist + if let Some(parent) = dest_path.parent() { + fs::create_dir_all(parent)?; + } + + // Download the file + let response = reqwest::get(URL.replace("{}", &i.to_string())).await?; + let bytes = response.bytes().await?; + + // Write to file + let mut file = fs::File::create(&dest_path)?; + file.write_all(&bytes)?; + + println!("Downloaded to {}", dest_path.display()); + + Ok(()) +} + +async fn download_partitioned( + dest_path: PathBuf, + range: Range, +) -> Result<(), Box> { + let mut join_set = JoinSet::new(); + for i in range { + let dest_path = dest_path.clone(); + join_set.spawn(async move { + download_benchmark(dest_path.join("hits").join(format!("{i}.parquet")), i).await + }); + } + join_set.join_all().await; + Ok(()) +} + +pub async fn generate_clickbench_data( + dest_path: &Path, + range: Range, +) -> Result<(), Box> { + download_partitioned(dest_path.to_path_buf(), range).await?; + Ok(()) +} + +pub async fn register_tables( + ctx: &SessionContext, + data_path: &Path, +) -> Result<(), DataFusionError> { + for entry in fs::read_dir(data_path)? { + let path = entry?.path(); + if path.is_dir() { + let table_name = path.file_name().unwrap().to_str().unwrap(); + ctx.register_parquet( + table_name, + path.to_str().unwrap(), + ParquetReadOptions::default(), + ) + .await?; + } + } + Ok(()) +} diff --git a/src/test_utils/mod.rs b/src/test_utils/mod.rs index 5f0efe22..7b033e12 100644 --- a/src/test_utils/mod.rs +++ b/src/test_utils/mod.rs @@ -1,3 +1,4 @@ +pub mod clickbench; pub mod in_memory_channel_resolver; pub mod insta; pub mod localhost; diff --git a/testdata/clickbench/queries/q0.sql b/testdata/clickbench/queries/q0.sql new file mode 100644 index 00000000..35f2b32e --- /dev/null +++ b/testdata/clickbench/queries/q0.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 + +-- set datafusion.execution.parquet.binary_as_string = true +SELECT COUNT(*) FROM hits; diff --git a/testdata/clickbench/queries/q1.sql b/testdata/clickbench/queries/q1.sql new file mode 100644 index 00000000..0bee959e --- /dev/null +++ b/testdata/clickbench/queries/q1.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT COUNT(*) FROM hits WHERE "AdvEngineID" <> 0; diff --git a/testdata/clickbench/queries/q10.sql b/testdata/clickbench/queries/q10.sql new file mode 100644 index 00000000..0f911480 --- /dev/null +++ b/testdata/clickbench/queries/q10.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "MobilePhoneModel", COUNT(DISTINCT "UserID") AS u FROM hits WHERE "MobilePhoneModel" <> '' GROUP BY "MobilePhoneModel" ORDER BY u DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q11.sql b/testdata/clickbench/queries/q11.sql new file mode 100644 index 00000000..bed8bb21 --- /dev/null +++ b/testdata/clickbench/queries/q11.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "MobilePhone", "MobilePhoneModel", COUNT(DISTINCT "UserID") AS u FROM hits WHERE "MobilePhoneModel" <> '' GROUP BY "MobilePhone", "MobilePhoneModel" ORDER BY u DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q12.sql b/testdata/clickbench/queries/q12.sql new file mode 100644 index 00000000..8cf09c00 --- /dev/null +++ b/testdata/clickbench/queries/q12.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "SearchPhrase", COUNT(*) AS c FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q13.sql b/testdata/clickbench/queries/q13.sql new file mode 100644 index 00000000..ef6583c8 --- /dev/null +++ b/testdata/clickbench/queries/q13.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "SearchPhrase", COUNT(DISTINCT "UserID") AS u FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY u DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q14.sql b/testdata/clickbench/queries/q14.sql new file mode 100644 index 00000000..dd267146 --- /dev/null +++ b/testdata/clickbench/queries/q14.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "SearchEngineID", "SearchPhrase", COUNT(*) AS c FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchEngineID", "SearchPhrase" ORDER BY c DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q15.sql b/testdata/clickbench/queries/q15.sql new file mode 100644 index 00000000..721d924c --- /dev/null +++ b/testdata/clickbench/queries/q15.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "UserID", COUNT(*) FROM hits GROUP BY "UserID" ORDER BY COUNT(*) DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q16.sql b/testdata/clickbench/queries/q16.sql new file mode 100644 index 00000000..389725d5 --- /dev/null +++ b/testdata/clickbench/queries/q16.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "UserID", "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", "SearchPhrase" ORDER BY COUNT(*) DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q17.sql b/testdata/clickbench/queries/q17.sql new file mode 100644 index 00000000..be9976a0 --- /dev/null +++ b/testdata/clickbench/queries/q17.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "UserID", "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", "SearchPhrase" LIMIT 10; diff --git a/testdata/clickbench/queries/q18.sql b/testdata/clickbench/queries/q18.sql new file mode 100644 index 00000000..d649f1ed --- /dev/null +++ b/testdata/clickbench/queries/q18.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "UserID", extract(minute FROM to_timestamp_seconds("EventTime")) AS m, "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", m, "SearchPhrase" ORDER BY COUNT(*) DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q19.sql b/testdata/clickbench/queries/q19.sql new file mode 100644 index 00000000..8212a765 --- /dev/null +++ b/testdata/clickbench/queries/q19.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "UserID" FROM hits WHERE "UserID" = 435090932899640449; diff --git a/testdata/clickbench/queries/q2.sql b/testdata/clickbench/queries/q2.sql new file mode 100644 index 00000000..bcdfad84 --- /dev/null +++ b/testdata/clickbench/queries/q2.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT SUM("AdvEngineID"), COUNT(*), AVG("ResolutionWidth") FROM hits; diff --git a/testdata/clickbench/queries/q20.sql b/testdata/clickbench/queries/q20.sql new file mode 100644 index 00000000..a7e488c2 --- /dev/null +++ b/testdata/clickbench/queries/q20.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT COUNT(*) FROM hits WHERE "URL" LIKE '%google%'; diff --git a/testdata/clickbench/queries/q21.sql b/testdata/clickbench/queries/q21.sql new file mode 100644 index 00000000..35516897 --- /dev/null +++ b/testdata/clickbench/queries/q21.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "SearchPhrase", MIN("URL"), COUNT(*) AS c FROM hits WHERE "URL" LIKE '%google%' AND "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q22.sql b/testdata/clickbench/queries/q22.sql new file mode 100644 index 00000000..d5f696e7 --- /dev/null +++ b/testdata/clickbench/queries/q22.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "SearchPhrase", MIN("URL"), MIN("Title"), COUNT(*) AS c, COUNT(DISTINCT "UserID") FROM hits WHERE "Title" LIKE '%Google%' AND "URL" NOT LIKE '%.google.%' AND "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q23.sql b/testdata/clickbench/queries/q23.sql new file mode 100644 index 00000000..ff399ded --- /dev/null +++ b/testdata/clickbench/queries/q23.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT * FROM hits WHERE "URL" LIKE '%google%' ORDER BY "EventTime" LIMIT 10; diff --git a/testdata/clickbench/queries/q24.sql b/testdata/clickbench/queries/q24.sql new file mode 100644 index 00000000..bc7a3641 --- /dev/null +++ b/testdata/clickbench/queries/q24.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "EventTime" LIMIT 10; diff --git a/testdata/clickbench/queries/q25.sql b/testdata/clickbench/queries/q25.sql new file mode 100644 index 00000000..5332e345 --- /dev/null +++ b/testdata/clickbench/queries/q25.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "SearchPhrase" LIMIT 10; diff --git a/testdata/clickbench/queries/q26.sql b/testdata/clickbench/queries/q26.sql new file mode 100644 index 00000000..bc1108ae --- /dev/null +++ b/testdata/clickbench/queries/q26.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "EventTime", "SearchPhrase" LIMIT 10; diff --git a/testdata/clickbench/queries/q27.sql b/testdata/clickbench/queries/q27.sql new file mode 100644 index 00000000..ba234d34 --- /dev/null +++ b/testdata/clickbench/queries/q27.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "CounterID", AVG(length("URL")) AS l, COUNT(*) AS c FROM hits WHERE "URL" <> '' GROUP BY "CounterID" HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; diff --git a/testdata/clickbench/queries/q28.sql b/testdata/clickbench/queries/q28.sql new file mode 100644 index 00000000..6a3bd037 --- /dev/null +++ b/testdata/clickbench/queries/q28.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT REGEXP_REPLACE("Referer", '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(length("Referer")) AS l, COUNT(*) AS c, MIN("Referer") FROM hits WHERE "Referer" <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; diff --git a/testdata/clickbench/queries/q29.sql b/testdata/clickbench/queries/q29.sql new file mode 100644 index 00000000..bca1eb7b --- /dev/null +++ b/testdata/clickbench/queries/q29.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT SUM("ResolutionWidth"), SUM("ResolutionWidth" + 1), SUM("ResolutionWidth" + 2), SUM("ResolutionWidth" + 3), SUM("ResolutionWidth" + 4), SUM("ResolutionWidth" + 5), SUM("ResolutionWidth" + 6), SUM("ResolutionWidth" + 7), SUM("ResolutionWidth" + 8), SUM("ResolutionWidth" + 9), SUM("ResolutionWidth" + 10), SUM("ResolutionWidth" + 11), SUM("ResolutionWidth" + 12), SUM("ResolutionWidth" + 13), SUM("ResolutionWidth" + 14), SUM("ResolutionWidth" + 15), SUM("ResolutionWidth" + 16), SUM("ResolutionWidth" + 17), SUM("ResolutionWidth" + 18), SUM("ResolutionWidth" + 19), SUM("ResolutionWidth" + 20), SUM("ResolutionWidth" + 21), SUM("ResolutionWidth" + 22), SUM("ResolutionWidth" + 23), SUM("ResolutionWidth" + 24), SUM("ResolutionWidth" + 25), SUM("ResolutionWidth" + 26), SUM("ResolutionWidth" + 27), SUM("ResolutionWidth" + 28), SUM("ResolutionWidth" + 29), SUM("ResolutionWidth" + 30), SUM("ResolutionWidth" + 31), SUM("ResolutionWidth" + 32), SUM("ResolutionWidth" + 33), SUM("ResolutionWidth" + 34), SUM("ResolutionWidth" + 35), SUM("ResolutionWidth" + 36), SUM("ResolutionWidth" + 37), SUM("ResolutionWidth" + 38), SUM("ResolutionWidth" + 39), SUM("ResolutionWidth" + 40), SUM("ResolutionWidth" + 41), SUM("ResolutionWidth" + 42), SUM("ResolutionWidth" + 43), SUM("ResolutionWidth" + 44), SUM("ResolutionWidth" + 45), SUM("ResolutionWidth" + 46), SUM("ResolutionWidth" + 47), SUM("ResolutionWidth" + 48), SUM("ResolutionWidth" + 49), SUM("ResolutionWidth" + 50), SUM("ResolutionWidth" + 51), SUM("ResolutionWidth" + 52), SUM("ResolutionWidth" + 53), SUM("ResolutionWidth" + 54), SUM("ResolutionWidth" + 55), SUM("ResolutionWidth" + 56), SUM("ResolutionWidth" + 57), SUM("ResolutionWidth" + 58), SUM("ResolutionWidth" + 59), SUM("ResolutionWidth" + 60), SUM("ResolutionWidth" + 61), SUM("ResolutionWidth" + 62), SUM("ResolutionWidth" + 63), SUM("ResolutionWidth" + 64), SUM("ResolutionWidth" + 65), SUM("ResolutionWidth" + 66), SUM("ResolutionWidth" + 67), SUM("ResolutionWidth" + 68), SUM("ResolutionWidth" + 69), SUM("ResolutionWidth" + 70), SUM("ResolutionWidth" + 71), SUM("ResolutionWidth" + 72), SUM("ResolutionWidth" + 73), SUM("ResolutionWidth" + 74), SUM("ResolutionWidth" + 75), SUM("ResolutionWidth" + 76), SUM("ResolutionWidth" + 77), SUM("ResolutionWidth" + 78), SUM("ResolutionWidth" + 79), SUM("ResolutionWidth" + 80), SUM("ResolutionWidth" + 81), SUM("ResolutionWidth" + 82), SUM("ResolutionWidth" + 83), SUM("ResolutionWidth" + 84), SUM("ResolutionWidth" + 85), SUM("ResolutionWidth" + 86), SUM("ResolutionWidth" + 87), SUM("ResolutionWidth" + 88), SUM("ResolutionWidth" + 89) FROM hits; diff --git a/testdata/clickbench/queries/q3.sql b/testdata/clickbench/queries/q3.sql new file mode 100644 index 00000000..09cdaca7 --- /dev/null +++ b/testdata/clickbench/queries/q3.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT AVG("UserID") FROM hits; diff --git a/testdata/clickbench/queries/q30.sql b/testdata/clickbench/queries/q30.sql new file mode 100644 index 00000000..c0d65792 --- /dev/null +++ b/testdata/clickbench/queries/q30.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "SearchEngineID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWidth") FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchEngineID", "ClientIP" ORDER BY c DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q31.sql b/testdata/clickbench/queries/q31.sql new file mode 100644 index 00000000..76ab3622 --- /dev/null +++ b/testdata/clickbench/queries/q31.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "WatchID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWidth") FROM hits WHERE "SearchPhrase" <> '' GROUP BY "WatchID", "ClientIP" ORDER BY c DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q32.sql b/testdata/clickbench/queries/q32.sql new file mode 100644 index 00000000..88f1e4ce --- /dev/null +++ b/testdata/clickbench/queries/q32.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "WatchID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWidth") FROM hits GROUP BY "WatchID", "ClientIP" ORDER BY c DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q33.sql b/testdata/clickbench/queries/q33.sql new file mode 100644 index 00000000..3740503b --- /dev/null +++ b/testdata/clickbench/queries/q33.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "URL", COUNT(*) AS c FROM hits GROUP BY "URL" ORDER BY c DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q34.sql b/testdata/clickbench/queries/q34.sql new file mode 100644 index 00000000..fdb7edbb --- /dev/null +++ b/testdata/clickbench/queries/q34.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT 1, "URL", COUNT(*) AS c FROM hits GROUP BY 1, "URL" ORDER BY c DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q35.sql b/testdata/clickbench/queries/q35.sql new file mode 100644 index 00000000..de7e2256 --- /dev/null +++ b/testdata/clickbench/queries/q35.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "ClientIP", "ClientIP" - 1, "ClientIP" - 2, "ClientIP" - 3, COUNT(*) AS c FROM hits GROUP BY "ClientIP", "ClientIP" - 1, "ClientIP" - 2, "ClientIP" - 3 ORDER BY c DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q36.sql b/testdata/clickbench/queries/q36.sql new file mode 100644 index 00000000..81b1199b --- /dev/null +++ b/testdata/clickbench/queries/q36.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "URL", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND "URL" <> '' GROUP BY "URL" ORDER BY PageViews DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q37.sql b/testdata/clickbench/queries/q37.sql new file mode 100644 index 00000000..fa4b85ff --- /dev/null +++ b/testdata/clickbench/queries/q37.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "Title", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND "Title" <> '' GROUP BY "Title" ORDER BY PageViews DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q38.sql b/testdata/clickbench/queries/q38.sql new file mode 100644 index 00000000..18fafab6 --- /dev/null +++ b/testdata/clickbench/queries/q38.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "URL", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "IsRefresh" = 0 AND "IsLink" <> 0 AND "IsDownload" = 0 GROUP BY "URL" ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; diff --git a/testdata/clickbench/queries/q39.sql b/testdata/clickbench/queries/q39.sql new file mode 100644 index 00000000..306f0caa --- /dev/null +++ b/testdata/clickbench/queries/q39.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "TraficSourceID", "SearchEngineID", "AdvEngineID", CASE WHEN ("SearchEngineID" = 0 AND "AdvEngineID" = 0) THEN "Referer" ELSE '' END AS Src, "URL" AS Dst, COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "IsRefresh" = 0 GROUP BY "TraficSourceID", "SearchEngineID", "AdvEngineID", Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; diff --git a/testdata/clickbench/queries/q4.sql b/testdata/clickbench/queries/q4.sql new file mode 100644 index 00000000..d89ca78c --- /dev/null +++ b/testdata/clickbench/queries/q4.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT COUNT(DISTINCT "UserID") FROM hits; diff --git a/testdata/clickbench/queries/q40.sql b/testdata/clickbench/queries/q40.sql new file mode 100644 index 00000000..e9d27f59 --- /dev/null +++ b/testdata/clickbench/queries/q40.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "URLHash", "EventDate", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "IsRefresh" = 0 AND "TraficSourceID" IN (-1, 6) AND "RefererHash" = 3594120000172545465 GROUP BY "URLHash", "EventDate" ORDER BY PageViews DESC LIMIT 10 OFFSET 100; diff --git a/testdata/clickbench/queries/q41.sql b/testdata/clickbench/queries/q41.sql new file mode 100644 index 00000000..0e067e2d --- /dev/null +++ b/testdata/clickbench/queries/q41.sql @@ -0,0 +1,3 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true +SELECT "WindowClientWidth", "WindowClientHeight", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "IsRefresh" = 0 AND "DontCountHits" = 0 AND "URLHash" = 2868770270353813622 GROUP BY "WindowClientWidth", "WindowClientHeight" ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; diff --git a/testdata/clickbench/queries/q42.sql b/testdata/clickbench/queries/q42.sql new file mode 100644 index 00000000..111cc1d3 --- /dev/null +++ b/testdata/clickbench/queries/q42.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT DATE_TRUNC('minute', to_timestamp_seconds("EventTime")) AS M, COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-14' AND "EventDate" <= '2013-07-15' AND "IsRefresh" = 0 AND "DontCountHits" = 0 GROUP BY DATE_TRUNC('minute', to_timestamp_seconds("EventTime")) ORDER BY DATE_TRUNC('minute', M) LIMIT 10 OFFSET 1000; diff --git a/testdata/clickbench/queries/q5.sql b/testdata/clickbench/queries/q5.sql new file mode 100644 index 00000000..d371cfb6 --- /dev/null +++ b/testdata/clickbench/queries/q5.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT COUNT(DISTINCT "SearchPhrase") FROM hits; diff --git a/testdata/clickbench/queries/q6.sql b/testdata/clickbench/queries/q6.sql new file mode 100644 index 00000000..5b4e896a --- /dev/null +++ b/testdata/clickbench/queries/q6.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT MIN("EventDate"), MAX("EventDate") FROM hits; diff --git a/testdata/clickbench/queries/q7.sql b/testdata/clickbench/queries/q7.sql new file mode 100644 index 00000000..afffcb13 --- /dev/null +++ b/testdata/clickbench/queries/q7.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "AdvEngineID", COUNT(*) FROM hits WHERE "AdvEngineID" <> 0 GROUP BY "AdvEngineID" ORDER BY COUNT(*) DESC; diff --git a/testdata/clickbench/queries/q8.sql b/testdata/clickbench/queries/q8.sql new file mode 100644 index 00000000..097880a9 --- /dev/null +++ b/testdata/clickbench/queries/q8.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "RegionID", COUNT(DISTINCT "UserID") AS u FROM hits GROUP BY "RegionID" ORDER BY u DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q9.sql b/testdata/clickbench/queries/q9.sql new file mode 100644 index 00000000..cb1b79bf --- /dev/null +++ b/testdata/clickbench/queries/q9.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "RegionID", SUM("AdvEngineID"), COUNT(*) AS c, AVG("ResolutionWidth"), COUNT(DISTINCT "UserID") FROM hits GROUP BY "RegionID" ORDER BY c DESC LIMIT 10; diff --git a/tests/clickbench_correctness_test.rs b/tests/clickbench_correctness_test.rs new file mode 100644 index 00000000..daae5143 --- /dev/null +++ b/tests/clickbench_correctness_test.rs @@ -0,0 +1,316 @@ +#[cfg(all(feature = "integration", feature = "clickbench", test))] +mod tests { + use datafusion::arrow::array::RecordBatch; + use datafusion::common::plan_err; + use datafusion::error::Result; + use datafusion::physical_plan::{ExecutionPlan, collect}; + use datafusion::prelude::SessionContext; + use datafusion_distributed::test_utils::clickbench; + use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::test_utils::property_based::{ + compare_ordering, compare_result_set, + }; + use datafusion_distributed::{ + DefaultSessionBuilder, DistributedExec, DistributedExt, display_plan_ascii, + }; + use std::ops::Range; + use std::path::Path; + use std::sync::Arc; + use tokio::sync::OnceCell; + + const NUM_WORKERS: usize = 4; + const FILES_PER_TASK: usize = 2; + const CARDINALITY_TASK_COUNT_FACTOR: f64 = 2.0; + const FILE_RANGE: Range = 0..3; + + #[tokio::test] + #[ignore = "Query 0 did not get distributed"] + async fn test_clickbench_0() -> Result<()> { + test_clickbench_query(0).await + } + + #[tokio::test] + async fn test_clickbench_1() -> Result<()> { + test_clickbench_query(1).await + } + + #[tokio::test] + async fn test_clickbench_2() -> Result<()> { + test_clickbench_query(2).await + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 1, Right set size: 1\n\nRows only in left (1 total):\n 2533767602294735360.00\n\nRows only in right (1 total):\n 2533767602294735872.00.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_3() -> Result<()> { + test_clickbench_query(3).await + } + + #[tokio::test] + async fn test_clickbench_4() -> Result<()> { + test_clickbench_query(4).await + } + + #[tokio::test] + async fn test_clickbench_5() -> Result<()> { + test_clickbench_query(5).await + } + + #[tokio::test] + #[ignore = "Query 6 did not get distributed"] + async fn test_clickbench_6() -> Result<()> { + test_clickbench_query(6).await + } + + #[tokio::test] + async fn test_clickbench_7() -> Result<()> { + test_clickbench_query(7).await + } + + #[tokio::test] + async fn test_clickbench_8() -> Result<()> { + test_clickbench_query(8).await + } + + #[tokio::test] + async fn test_clickbench_9() -> Result<()> { + test_clickbench_query(9).await + } + + #[tokio::test] + async fn test_clickbench_10() -> Result<()> { + test_clickbench_query(10).await + } + + #[tokio::test] + async fn test_clickbench_11() -> Result<()> { + test_clickbench_query(11).await + } + + #[tokio::test] + async fn test_clickbench_12() -> Result<()> { + test_clickbench_query(12).await + } + + #[tokio::test] + async fn test_clickbench_13() -> Result<()> { + test_clickbench_query(13).await + } + + #[tokio::test] + async fn test_clickbench_14() -> Result<()> { + test_clickbench_query(14).await + } + + #[tokio::test] + async fn test_clickbench_15() -> Result<()> { + test_clickbench_query(15).await + } + + #[tokio::test] + async fn test_clickbench_16() -> Result<()> { + test_clickbench_query(16).await + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (10 total):\n 3219866204653196665||4\n 3220056705148678697||11\n 3221898002592879542||1\n 3223026783585713477||23\n 3223839745005575457||116\n 3223839745005575457|d0bcd0bed0b6d0bdd0be20d0bbd0b820d0b220d0bad180d0bed0bad0bed0b4d0b8d180d0bed0b2d0b5d0bbd18cd18820d0b1d180d0bed0b4d18b20d0bdd0b020d181d182d0bed0bbd18b20d0b2d0be20d0b2d0bbd0b0d0b4d0b8d0b2d0bed181d182d0bed0ba20d0b2d0b2d0be|1\n 3223949769615485893||1\n 3226415756450197918||24\n 3226664959488084815||62\n 3227160743723019373||71\n\nRows only in right (10 total):\n 700182585509527889||2\n 724127359630680276|d0b8d0b3d180d18b20d0b820d181d0b5d0b3d0bed0b4d0bdd18f3f|1\n 766120398574852544||1\n 766739966065297239||1\n 783205612738304865||3\n 797289180007803204||2\n 804968013253615745||1\n 830548852254311605||1\n 849024737642146119||1\n 849169469997862534||1.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_17() -> Result<()> { + test_clickbench_query(17).await + } + + #[tokio::test] + async fn test_clickbench_18() -> Result<()> { + test_clickbench_query(18).await + } + + #[tokio::test] + async fn test_clickbench_19() -> Result<()> { + test_clickbench_query(19).await + } + + #[tokio::test] + async fn test_clickbench_20() -> Result<()> { + test_clickbench_query(20).await + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (5 total):\n d181d0bbd0b0d0b2d0bbd18fd182d18c20d0bfd0bed180d0bed0b4d0b8d182d181d18f20d0bed182d0b5d0bbd0b8203230313320d181d0bcd0bed182d180d0b5d182d18c|687474703a253246253246766b2e636f6d2e75612f676f6f676c652d6a61726b6f76736b6179612d4c697065636b64|1\n d0b1d0b0d0bdd0bad0bed0bcd0b0d182d0b5d180d0b8d0b0d0bbd18b20d181d0bcd0bed182d180d0b5d182d18c|687474703a2f2f6f72656e627572672e6972722e72752532466b7572746b692532462532467777772e676f6f676c652e72752f6d617a64612d332d6b6f6d6e2d6b762d4b617a616e2e74757475746f72736b2f64657461696c|1\n d0bcd0bed0bdd0b8d182d18c20d0bad0b0d0bad0bed0b520d0bed0b7d0b5d180d0b0|687474703a2f2f6175746f2e7269612e75612f6175746f5f69643d30266f726465723d46616c7365266d696e707269782e72752f6b617465676f726979612f767369652d646c69612d647275676f652f6d61746572696e7374766f2f676f6f676c652d706f6c697331343334343532|1\n d181d0bad0b0d187d0b0d182d18c20d0b4d0b5d0bdd0b5d0b320d181d183d180d0b3d183d182|687474703a2f2f7469656e736b6169612d6d6f64612d627269657469656c6b612d6b6f736b6f76736b2f64657461696c2e676f6f676c65|1\n d0b220d0b0d0b2d0b3d183d181d1822032343720d0b3d180d183d181d182d0b8d0bcd0bed188d0bad0b020d0bdd0b020d0bad180d0b8d181d182d180d0b0d182|687474703a2f2f7469656e736b6169612d6d6f64612d627269756b692f676f6f676c652e72752f7e61706f6b2e72752f635f312d755f313138383839352c39373536|1\n\nRows only in right (5 total):\n d0bcd0bed0b4d0b5d0bad18120d183d0bbd0b8d186d0b5d0bdd0b7d0b8d0bdd0bed0b2d0b020d0b3d0bed0b2d18fd0b4d0b8d0bdd0b0|687474703a2f2f73616d6172612e6972722e72752f636174616c6f675f676f6f676c652d636865726e796a2d393233353636363635372f3f64617465|1\n d0bbd0b0d0b2d0bfd0bbd0b0d0bdd188d0b5d182d0bdd0b8d18520d183d181d0bbd0bed0b2d0b0d0bcd0b820d0b2d181d0b520d181d0b5d180d0b8d0b820d0b4d0b0d182d0b020d186d0b5d0bcd0b5d0bdd0b8|687474703a2f2f73616d6172612e6972722e72752f636174616c6f675f676f6f676c654d425225323661642533443930253236707a|1\n d0bad0b0d0ba20d0bfd180d0bed0b4d0b0d0bcd0b820d0b4d0bbd18f20d0b4d0b5d0b2d183d188d0bad0b8|687474703a253246253246777777772e626f6e707269782e7275253235326625323532663737363925323532663131303931392d6c65766f652d676f6f676c652d7368746f72792e72752f666f72756d2f666f72756d2e6d617465722e72752f6461696c792f63616c63756c61746f72|1\n d0b6d0b0d180d0b5d0bdd18cd18f20d0b32ed181d183d180d0bed0b2d0b0d0bdd0b8d0b520d0b2d0bed180d0bed0bdd0b5d0b6d181d0bad0b0d18f20d0bed0b1d0bbd0b0d181d182d0bed0bfd180d0b8d0bbd0b520d0bfd0bed181d0bbd0b5d0b4d0bdd0b8d0b520d0bad0bed181d18b|687474703a2f2f756b7261696e627572672f65636f2d6d6c656b2f65636f6e646172792f73686f77746f7069632e7068703f69643d3436333837362e68746d6c3f69643d32303634313333363631253246676f6f676c652d4170706c655765624b69742532463533372e333620284b48544d4c2c206c696b65|1\n d180d0b8d0be20d0bdd0b020d0bad0b0d180d182d0bed187d0bdd0b8d186d0b020d181d0bcd0bed182d180d0b5d182d18c20d0bed0bdd0bbd0b0d0b9d0bd|687474703a2f2f73616d6172612e6972722e72752f636174616c6f675f676f6f676c654d425225323661642533443930253236707a|1.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_21() -> Result<()> { + test_clickbench_query(21).await + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (1 total):\n d0bad0b0d0bad0bed0b920d0bfd0bbd0bed189d0b0d0b4d0bad0b8d0bcd0b820d0b4d0bed181d182d0b0d0b2d0bad0b8|687474703a253246253246766b2e636f6d2f696672616d652d6f77612e68746d6c3f313d31266369643d353737266f6b693d31266f705f63617465676f72795f69645d3d332673656c656374|d092d0b0d0bad0b0d0bdd181d0b8d18f20d091d0a0d090d09ad090d09d20d090d09dd094d0a0d095d0a1202d20d0bfd0bed0bfd0b0d0bbd0b820d0bad183d0bfd0b8d182d18c20d0b4d0bed0bcd0bed0b5d187d0bdd18bd0b520d188d0bad0b0d184d0b020476f6f676c652e636f6d203a3a20d0bad0bed182d182d0b5d0bad181d1822c20d091d183d180d18fd182d0bdd0b8d0bad0b820d0b4d0bbd18f20d0bfd0b5d187d18c20d0bcd0b5d0b1d0b5d0bbd18cd0b520d0b4d0bbd18f20d0b4d0b5d0b2d183d188d0bad0b0|5|1\n\nRows only in right (1 total):\n d0bad0bed0bfd182d0b8d0bcd0b8d0bad0b2d0b8d0b4d18b20d18ed180d0b8d0b920d0bfd0bed181d0bbd0b5d0b4d0bdd18fd18f|68747470733a2f2f70726f64756b747925324670756c6f76652e72752f626f6f6b6c79617474696f6e2d7761722d73696e696a2d393430343139342c3936323435332f666f746f|d09bd0b5d0b3d0bad0be20d0bdd0b020d183d187d0b0d181d182d0bdd18bd0b520d183d187d0b0d181d182d0bdd0b8d0bad0bed0b22e2c20d0a6d0b5d0bdd18b202d20d0a1d182d0b8d0bbd18cd0bdd0b0d18f20d0bfd0b0d180d0bdd0b5d0bc2e20d0a1d0b0d0b3d0b0d0bdd180d0bed0b320d0b4d0bed0b3d0b0d0b4d0b5d0bdd0b8d18f203a20d0a2d183d180d186d0b8d0b82c20d0bad183d0bfd0b8d182d18c20d18320313020d0b4d0bdd0b520d0bad0bed0bbd18cd0bdd18bd0b520d0bcd0b0d188d0b8d0bdd0bad0b820d0bdd0b520d0bfd180d0b5d0b4d181d182d0b0d0b2d0bad0b8202d20d09dd0bed0b2d0b0d18f20d18120d0b8d0b7d0b1d0b8d0b5d0bdd0b8d0b520d181d0bfd180d0bed0b4d0b0d0b6d0b03a20d0bad0bed182d18fd182d0b0203230313420d0b32ed0b22e20d0a6d0b5d0bdd0b03a2034373530302d313045434f30363020e28093202d2d2d2d2d2d2d2d20d0bad183d0bfd0b8d182d18c20d0bad0b2d0b0d180d182d0b8d180d18320d09ed180d0b5d0bdd0b1d183d180d0b32028d0a0d0bed181d181d0b8d0b82047616c616e7472617820466c616d696c6961646120476f6f676c652c204ed0be20313820d184d0bed182d0bed0bad0bed0bdd0b2d0b5d180d0ba20d0a1d183d0bfd0b5d18020d09ad0b0d180d0b4d0b8d0b3d0b0d0bd|5|1.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_22() -> Result<()> { + test_clickbench_query(22).await + } + + #[tokio::test] + async fn test_clickbench_23() -> Result<()> { + test_clickbench_query(23).await + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (1 total):\n d0b2d181d0bfd0bed0bcd0bdd0b8d182d18c20d181d0bed0bbd0bdd0b5d0bdd0b8d0b520d0b1d0b0d0bdd0bad0b020d0bbd0b0d0b420d184d0b8d0bbd18cd0bc\n\nRows only in right (1 total):\n d0bed182d0b2d0bed0b4d0b020d0b4d0bbd18f20d0bfd0b8d180d0bed0b6d0bad0b820d0bbd0b5d187d0b5d0bdd0bdd18b20d0b2d181d0b520d181d0b5d180d196d197.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_24() -> Result<()> { + test_clickbench_query(24).await + } + + #[tokio::test] + async fn test_clickbench_25() -> Result<()> { + test_clickbench_query(25).await + } + + #[tokio::test] + async fn test_clickbench_26() -> Result<()> { + test_clickbench_query(26).await + } + + #[tokio::test] + async fn test_clickbench_27() -> Result<()> { + test_clickbench_query(27).await + } + + #[tokio::test] + async fn test_clickbench_28() -> Result<()> { + test_clickbench_query(28).await + } + + #[tokio::test] + async fn test_clickbench_29() -> Result<()> { + test_clickbench_query(29).await + } + + #[tokio::test] + async fn test_clickbench_30() -> Result<()> { + test_clickbench_query(30).await + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (10 total):\n 8673025726158767406|1264438551|1|0|1990.00\n 5320052218057629211|-1703087277|1|0|1996.00\n 6244273852606083750|1554672832|1|0|1638.00\n 8628753750962053665|1215278356|1|0|1087.00\n 7035318163404387241|1326714320|1|0|1638.00\n 8431857775494210873|1237512945|1|0|1996.00\n 5110752526539992124|37611695|1|0|1917.00\n 8986794334343068049|1860752926|1|0|1638.00\n 8044147848299485837|1382122372|1|0|1368.00\n 7936057634954670727|1897481896|1|0|1638.00\n\nRows only in right (10 total):\n 5132615111782210132|-50313020|1|0|1368.00\n 5783789691451717551|-1310327384|1|0|1638.00\n 5756260993772351383|1484317883|1|0|375.00\n 7739310142000732364|991864113|1|0|1368.00\n 7593472904893539271|-151291403|1|0|1087.00\n 6339599967989898410|1543815587|1|0|1638.00\n 7794346560421945218|1645556180|1|0|1368.00\n 6112645108657361792|593586188|1|0|1638.00\n 6675910710751922756|-816379256|1|0|1368.00\n 5802727636196431835|1986422271|1|0|1996.00.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_31() -> Result<()> { + test_clickbench_query(31).await + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (10 total):\n 7643059318918524417|1767085700|1|0|0.00\n 5437163248266133938|-1465369615|1|0|0.00\n 9142541582422390102|-1465369615|1|0|0.00\n 8438994503411842126|-1465369615|1|0|0.00\n 7362096505818029859|-565678477|1|0|0.00\n 4928022308880516715|1699955284|1|0|0.00\n 5269769817689282522|1699955284|1|0|0.00\n 9081648050908046886|1699955284|1|0|0.00\n 6824181869275536503|1699955284|1|0|0.00\n 6905712404475757487|1552811156|1|0|0.00\n\nRows only in right (10 total):\n 6967277596165459879|-941091661|1|0|1368.00\n 5796237228224217668|-1310327384|1|0|1638.00\n 7218628137278606666|1511490240|1|0|1638.00\n 8314760197723815280|1566105210|1|0|1996.00\n 7053263954762394007|757778490|1|0|339.00\n 6283334114093174531|1216031795|1|0|1368.00\n 8818295356247036741|83042182|1|0|1638.00\n 6620528864937282562|-862894777|1|0|1996.00\n 8466121050002905379|83042182|1|0|1638.00\n 7554844936512227411|-1746904856|1|0|1368.00.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_32() -> Result<()> { + test_clickbench_query(32).await + } + + #[tokio::test] + async fn test_clickbench_33() -> Result<()> { + test_clickbench_query(33).await + } + + #[tokio::test] + async fn test_clickbench_34() -> Result<()> { + test_clickbench_query(34).await + } + + #[tokio::test] + async fn test_clickbench_35() -> Result<()> { + test_clickbench_query(35).await + } + + #[tokio::test] + async fn test_clickbench_36() -> Result<()> { + test_clickbench_query(36).await + } + + #[tokio::test] + async fn test_clickbench_37() -> Result<()> { + test_clickbench_query(37).await + } + + #[tokio::test] + async fn test_clickbench_38() -> Result<()> { + test_clickbench_query(38).await + } + + #[tokio::test] + async fn test_clickbench_39() -> Result<()> { + test_clickbench_query(39).await + } + + #[tokio::test] + async fn test_clickbench_40() -> Result<()> { + test_clickbench_query(40).await + } + + #[tokio::test] + async fn test_clickbench_41() -> Result<()> { + test_clickbench_query(41).await + } + + #[tokio::test] + #[ignore = "ordering mismatch: expected ordering: Some(LexOrdering { exprs: [PhysicalSortExpr { expr: ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }, options: SortOptions { descending: false, nulls_first: false } }], set: {ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }} }), actual ordering: Some(LexOrdering { exprs: [PhysicalSortExpr { expr: ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }, options: SortOptions { descending: false, nulls_first: false } }], set: {ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }} })"] + async fn test_clickbench_42() -> Result<()> { + test_clickbench_query(42).await + } + + static INIT_TEST_TPCDS_TABLES: OnceCell<()> = OnceCell::const_new(); + + async fn run( + ctx: &SessionContext, + query_sql: &str, + ) -> (Arc, Arc>>) { + let df = ctx.sql(query_sql).await.unwrap(); + let task_ctx = ctx.task_ctx(); + let plan = df.create_physical_plan().await.unwrap(); + (plan.clone(), Arc::new(collect(plan, task_ctx).await)) // Collect execution errors, do not unwrap. + } + + async fn test_clickbench_query(query_id: usize) -> Result<()> { + let data_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(format!( + "testdata/clickbench/correctness_range{}-{}", + FILE_RANGE.start, FILE_RANGE.end + )); + INIT_TEST_TPCDS_TABLES + .get_or_init(|| async { + clickbench::generate_clickbench_data(&data_dir, FILE_RANGE) + .await + .unwrap(); + }) + .await; + + let query_sql = clickbench::get_test_clickbench_query(query_id)?; + // Create a single node context to compare results to. + let s_ctx = SessionContext::new(); + + // Make distributed localhost context to run queries + let (d_ctx, _guard) = start_localhost_context(NUM_WORKERS, DefaultSessionBuilder).await; + let d_ctx = d_ctx + .with_distributed_files_per_task(FILES_PER_TASK)? + .with_distributed_cardinality_effect_task_scale_factor(CARDINALITY_TASK_COUNT_FACTOR)?; + + clickbench::register_tables(&s_ctx, &data_dir).await?; + clickbench::register_tables(&d_ctx, &data_dir).await?; + + let (s_plan, s_results) = run(&s_ctx, &query_sql).await; + let (d_plan, d_results) = run(&d_ctx, &query_sql).await; + + if !d_plan.as_any().is::() { + return plan_err!("Query {query_id} did not get distributed"); + } + let display = display_plan_ascii(d_plan.as_ref(), false); + println!("Query {query_id}:\n{display}"); + + let compare_result_set = { + let d_results = d_results.clone(); + let s_results = s_results.clone(); + tokio::task::spawn_blocking(move || async move { + compare_result_set(&d_results, &s_results) + }) + }; + let compare_ordering = { + let d_results = d_results.clone(); + tokio::task::spawn_blocking(move || async move { + compare_ordering(d_plan, s_plan, &d_results) + }) + }; + compare_result_set.await.unwrap().await?; + compare_ordering.await.unwrap().await?; + + Ok(()) + } +} diff --git a/tests/clickbench_plans_test.rs b/tests/clickbench_plans_test.rs new file mode 100644 index 00000000..907c51d2 --- /dev/null +++ b/tests/clickbench_plans_test.rs @@ -0,0 +1,942 @@ +#[cfg(all(feature = "integration", feature = "clickbench", test))] +mod tests { + use datafusion::error::Result; + use datafusion_distributed::test_utils::clickbench; + use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::{ + DefaultSessionBuilder, DistributedExec, DistributedExt, assert_snapshot, display_plan_ascii, + }; + use std::ops::Range; + use std::path::Path; + use tokio::sync::OnceCell; + + const NUM_WORKERS: usize = 4; + const FILES_PER_TASK: usize = 2; + const CARDINALITY_TASK_COUNT_FACTOR: f64 = 1.5; + const FILE_RANGE: Range = 0..3; + + #[tokio::test] + #[ignore = "Query 0 did not get distributed"] + async fn test_clickbench_0() -> Result<()> { + let display = test_clickbench_query(0).await?; + assert_snapshot!(display, @""); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_1() -> Result<()> { + let display = test_clickbench_query(1).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: AdvEngineID@0 != 0 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[AdvEngineID], file_type=parquet, predicate=AdvEngineID@0 != 0, pruning_predicate=AdvEngineID_null_count@2 != row_count@3 AND (AdvEngineID_min@0 != 0 OR 0 != AdvEngineID_max@1), required_guarantees=[AdvEngineID not in (0)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_2() -> Result<()> { + let display = test_clickbench_query(2).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[sum(hits.AdvEngineID)@0 as sum(hits.AdvEngineID), count(Int64(1))@1 as count(*), avg(hits.ResolutionWidth)@2 as avg(hits.ResolutionWidth)] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(hits.AdvEngineID), count(Int64(1)), avg(hits.ResolutionWidth)] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(hits.AdvEngineID), count(Int64(1)), avg(hits.ResolutionWidth)] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[ResolutionWidth, AdvEngineID], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 1, Right set size: 1\n\nRows only in left (1 total):\n 2533767602294735360.00\n\nRows only in right (1 total):\n 2533767602294735872.00.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_3() -> Result<()> { + let display = test_clickbench_query(3).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(hits.UserID)] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(hits.UserID)] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_4() -> Result<()> { + let display = test_clickbench_query(4).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(alias1)@0 as count(DISTINCT hits.UserID)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(alias1)] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(alias1)] + │ AggregateExec: mode=FinalPartitioned, gby=[alias1@0 as alias1], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([alias1@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[UserID@0 as alias1], aggr=[] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_5() -> Result<()> { + let display = test_clickbench_query(5).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(alias1)@0 as count(DISTINCT hits.SearchPhrase)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(alias1)] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(alias1)] + │ AggregateExec: mode=FinalPartitioned, gby=[alias1@0 as alias1], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([alias1@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[SearchPhrase@0 as alias1], aggr=[] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchPhrase], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + #[ignore = "Query 6 did not get distributed"] + async fn test_clickbench_6() -> Result<()> { + let display = test_clickbench_query(6).await?; + assert_snapshot!(display, @""); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_7() -> Result<()> { + let display = test_clickbench_query(7).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[AdvEngineID@0 as AdvEngineID, count(*)@1 as count(*)] + │ SortPreservingMergeExec: [count(Int64(1))@2 DESC] + │ SortExec: expr=[count(*)@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[AdvEngineID@0 as AdvEngineID, count(Int64(1))@1 as count(*), count(Int64(1))@1 as count(Int64(1))] + │ AggregateExec: mode=FinalPartitioned, gby=[AdvEngineID@0 as AdvEngineID], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([AdvEngineID@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[AdvEngineID@0 as AdvEngineID], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: AdvEngineID@0 != 0 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[AdvEngineID], file_type=parquet, predicate=AdvEngineID@0 != 0, pruning_predicate=AdvEngineID_null_count@2 != row_count@3 AND (AdvEngineID_min@0 != 0 OR 0 != AdvEngineID_max@1), required_guarantees=[AdvEngineID not in (0)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_8() -> Result<()> { + let display = test_clickbench_query(8).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [u@1 DESC], fetch=10 + │ SortExec: TopK(fetch=10), expr=[u@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[RegionID@0 as RegionID, count(alias1)@1 as u] + │ AggregateExec: mode=FinalPartitioned, gby=[RegionID@0 as RegionID], aggr=[count(alias1)] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([RegionID@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[RegionID@0 as RegionID], aggr=[count(alias1)] + │ AggregateExec: mode=FinalPartitioned, gby=[RegionID@0 as RegionID, alias1@1 as alias1], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([RegionID@0, alias1@1], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[RegionID@0 as RegionID, UserID@1 as alias1], aggr=[] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[RegionID, UserID], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_9() -> Result<()> { + let display = test_clickbench_query(9).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c@2 DESC], fetch=10 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[RegionID@0 as RegionID, sum(hits.AdvEngineID)@1 as sum(hits.AdvEngineID), count(Int64(1))@2 as c, avg(hits.ResolutionWidth)@3 as avg(hits.ResolutionWidth), count(DISTINCT hits.UserID)@4 as count(DISTINCT hits.UserID)] + │ AggregateExec: mode=FinalPartitioned, gby=[RegionID@0 as RegionID], aggr=[sum(hits.AdvEngineID), count(Int64(1)), avg(hits.ResolutionWidth), count(DISTINCT hits.UserID)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([RegionID@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[RegionID@0 as RegionID], aggr=[sum(hits.AdvEngineID), count(Int64(1)), avg(hits.ResolutionWidth), count(DISTINCT hits.UserID)] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[RegionID, UserID, ResolutionWidth, AdvEngineID], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_10() -> Result<()> { + let display = test_clickbench_query(10).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [u@1 DESC], fetch=10 + │ SortExec: TopK(fetch=10), expr=[u@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[MobilePhoneModel@0 as MobilePhoneModel, count(alias1)@1 as u] + │ AggregateExec: mode=FinalPartitioned, gby=[MobilePhoneModel@0 as MobilePhoneModel], aggr=[count(alias1)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([MobilePhoneModel@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[MobilePhoneModel@0 as MobilePhoneModel], aggr=[count(alias1)] + │ AggregateExec: mode=FinalPartitioned, gby=[MobilePhoneModel@0 as MobilePhoneModel, alias1@1 as alias1], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([MobilePhoneModel@0, alias1@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[MobilePhoneModel@1 as MobilePhoneModel, UserID@0 as alias1], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: MobilePhoneModel@1 != + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID, MobilePhoneModel], file_type=parquet, predicate=MobilePhoneModel@1 != , pruning_predicate=MobilePhoneModel_null_count@2 != row_count@3 AND (MobilePhoneModel_min@0 != OR != MobilePhoneModel_max@1), required_guarantees=[MobilePhoneModel not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_11() -> Result<()> { + let display = test_clickbench_query(11).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [u@2 DESC], fetch=10 + │ SortExec: TopK(fetch=10), expr=[u@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel, count(alias1)@2 as u] + │ AggregateExec: mode=FinalPartitioned, gby=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel], aggr=[count(alias1)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([MobilePhone@0, MobilePhoneModel@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel], aggr=[count(alias1)] + │ AggregateExec: mode=FinalPartitioned, gby=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel, alias1@2 as alias1], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([MobilePhone@0, MobilePhoneModel@1, alias1@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[MobilePhone@1 as MobilePhone, MobilePhoneModel@2 as MobilePhoneModel, UserID@0 as alias1], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: MobilePhoneModel@2 != + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID, MobilePhone, MobilePhoneModel], file_type=parquet, predicate=MobilePhoneModel@2 != , pruning_predicate=MobilePhoneModel_null_count@2 != row_count@3 AND (MobilePhoneModel_min@0 != OR != MobilePhoneModel_max@1), required_guarantees=[MobilePhoneModel not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_12() -> Result<()> { + let display = test_clickbench_query(12).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c@1 DESC], fetch=10 + │ SortExec: TopK(fetch=10), expr=[c@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase, count(Int64(1))@1 as c] + │ AggregateExec: mode=FinalPartitioned, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([SearchPhrase@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: SearchPhrase@0 != + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchPhrase], file_type=parquet, predicate=SearchPhrase@0 != , pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_13() -> Result<()> { + let display = test_clickbench_query(13).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [u@1 DESC], fetch=10 + │ SortExec: TopK(fetch=10), expr=[u@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase, count(alias1)@1 as u] + │ AggregateExec: mode=FinalPartitioned, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(alias1)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([SearchPhrase@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(alias1)] + │ AggregateExec: mode=FinalPartitioned, gby=[SearchPhrase@0 as SearchPhrase, alias1@1 as alias1], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([SearchPhrase@0, alias1@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[SearchPhrase@1 as SearchPhrase, UserID@0 as alias1], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: SearchPhrase@1 != + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID, SearchPhrase], file_type=parquet, predicate=SearchPhrase@1 != , pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_14() -> Result<()> { + let display = test_clickbench_query(14).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c@2 DESC], fetch=10 + │ SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[SearchEngineID@0 as SearchEngineID, SearchPhrase@1 as SearchPhrase, count(Int64(1))@2 as c] + │ AggregateExec: mode=FinalPartitioned, gby=[SearchEngineID@0 as SearchEngineID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([SearchEngineID@0, SearchPhrase@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[SearchEngineID@0 as SearchEngineID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: SearchPhrase@1 != + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchEngineID, SearchPhrase], file_type=parquet, predicate=SearchPhrase@1 != , pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_15() -> Result<()> { + let display = test_clickbench_query(15).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[UserID@0 as UserID, count(*)@1 as count(*)] + │ SortPreservingMergeExec: [count(Int64(1))@2 DESC], fetch=10 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[count(*)@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[UserID@0 as UserID, count(Int64(1))@1 as count(*), count(Int64(1))@1 as count(Int64(1))] + │ AggregateExec: mode=FinalPartitioned, gby=[UserID@0 as UserID], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([UserID@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[UserID@0 as UserID], aggr=[count(Int64(1))] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_16() -> Result<()> { + let display = test_clickbench_query(16).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[UserID@0 as UserID, SearchPhrase@1 as SearchPhrase, count(*)@2 as count(*)] + │ SortPreservingMergeExec: [count(Int64(1))@3 DESC], fetch=10 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[count(*)@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[UserID@0 as UserID, SearchPhrase@1 as SearchPhrase, count(Int64(1))@2 as count(*), count(Int64(1))@2 as count(Int64(1))] + │ AggregateExec: mode=FinalPartitioned, gby=[UserID@0 as UserID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([UserID@0, SearchPhrase@1], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[UserID@0 as UserID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID, SearchPhrase], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (10 total):\n 3219866204653196665||4\n 3220056705148678697||11\n 3221898002592879542||1\n 3223026783585713477||23\n 3223839745005575457||116\n 3223839745005575457|d0bcd0bed0b6d0bdd0be20d0bbd0b820d0b220d0bad180d0bed0bad0bed0b4d0b8d180d0bed0b2d0b5d0bbd18cd18820d0b1d180d0bed0b4d18b20d0bdd0b020d181d182d0bed0bbd18b20d0b2d0be20d0b2d0bbd0b0d0b4d0b8d0b2d0bed181d182d0bed0ba20d0b2d0b2d0be|1\n 3223949769615485893||1\n 3226415756450197918||24\n 3226664959488084815||62\n 3227160743723019373||71\n\nRows only in right (10 total):\n 700182585509527889||2\n 724127359630680276|d0b8d0b3d180d18b20d0b820d181d0b5d0b3d0bed0b4d0bdd18f3f|1\n 766120398574852544||1\n 766739966065297239||1\n 783205612738304865||3\n 797289180007803204||2\n 804968013253615745||1\n 830548852254311605||1\n 849024737642146119||1\n 849169469997862534||1.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_17() -> Result<()> { + let display = test_clickbench_query(17).await?; + assert_snapshot!(display, @""); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_18() -> Result<()> { + let display = test_clickbench_query(18).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[UserID@0 as UserID, m@1 as m, SearchPhrase@2 as SearchPhrase, count(*)@3 as count(*)] + │ SortPreservingMergeExec: [count(Int64(1))@4 DESC], fetch=10 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[count(*)@3 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[UserID@0 as UserID, date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime))@1 as m, SearchPhrase@2 as SearchPhrase, count(Int64(1))@3 as count(*), count(Int64(1))@3 as count(Int64(1))] + │ AggregateExec: mode=FinalPartitioned, gby=[UserID@0 as UserID, date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime))@1 as date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime)), SearchPhrase@2 as SearchPhrase], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([UserID@0, date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime))@1, SearchPhrase@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[UserID@1 as UserID, date_part(MINUTE, to_timestamp_seconds(EventTime@0)) as date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime)), SearchPhrase@2 as SearchPhrase], aggr=[count(Int64(1))] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventTime, UserID, SearchPhrase], file_type=parquet + └────────────────────────────────────────────────── + "#); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_19() -> Result<()> { + let display = test_clickbench_query(19).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: UserID@0 = 435090932899640449 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID], file_type=parquet, predicate=UserID@0 = 435090932899640449, pruning_predicate=UserID_null_count@2 != row_count@3 AND UserID_min@0 <= 435090932899640449 AND 435090932899640449 <= UserID_max@1, required_guarantees=[UserID in (435090932899640449)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_20() -> Result<()> { + let display = test_clickbench_query(20).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CAST(URL@0 AS Utf8View) LIKE %google% + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[URL], file_type=parquet, predicate=CAST(URL@0 AS Utf8View) LIKE %google% + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (5 total):\n d181d0bbd0b0d0b2d0bbd18fd182d18c20d0bfd0bed180d0bed0b4d0b8d182d181d18f20d0bed182d0b5d0bbd0b8203230313320d181d0bcd0bed182d180d0b5d182d18c|687474703a253246253246766b2e636f6d2e75612f676f6f676c652d6a61726b6f76736b6179612d4c697065636b64|1\n d0b1d0b0d0bdd0bad0bed0bcd0b0d182d0b5d180d0b8d0b0d0bbd18b20d181d0bcd0bed182d180d0b5d182d18c|687474703a2f2f6f72656e627572672e6972722e72752532466b7572746b692532462532467777772e676f6f676c652e72752f6d617a64612d332d6b6f6d6e2d6b762d4b617a616e2e74757475746f72736b2f64657461696c|1\n d0bcd0bed0bdd0b8d182d18c20d0bad0b0d0bad0bed0b520d0bed0b7d0b5d180d0b0|687474703a2f2f6175746f2e7269612e75612f6175746f5f69643d30266f726465723d46616c7365266d696e707269782e72752f6b617465676f726979612f767369652d646c69612d647275676f652f6d61746572696e7374766f2f676f6f676c652d706f6c697331343334343532|1\n d181d0bad0b0d187d0b0d182d18c20d0b4d0b5d0bdd0b5d0b320d181d183d180d0b3d183d182|687474703a2f2f7469656e736b6169612d6d6f64612d627269657469656c6b612d6b6f736b6f76736b2f64657461696c2e676f6f676c65|1\n d0b220d0b0d0b2d0b3d183d181d1822032343720d0b3d180d183d181d182d0b8d0bcd0bed188d0bad0b020d0bdd0b020d0bad180d0b8d181d182d180d0b0d182|687474703a2f2f7469656e736b6169612d6d6f64612d627269756b692f676f6f676c652e72752f7e61706f6b2e72752f635f312d755f313138383839352c39373536|1\n\nRows only in right (5 total):\n d0bcd0bed0b4d0b5d0bad18120d183d0bbd0b8d186d0b5d0bdd0b7d0b8d0bdd0bed0b2d0b020d0b3d0bed0b2d18fd0b4d0b8d0bdd0b0|687474703a2f2f73616d6172612e6972722e72752f636174616c6f675f676f6f676c652d636865726e796a2d393233353636363635372f3f64617465|1\n d0bbd0b0d0b2d0bfd0bbd0b0d0bdd188d0b5d182d0bdd0b8d18520d183d181d0bbd0bed0b2d0b0d0bcd0b820d0b2d181d0b520d181d0b5d180d0b8d0b820d0b4d0b0d182d0b020d186d0b5d0bcd0b5d0bdd0b8|687474703a2f2f73616d6172612e6972722e72752f636174616c6f675f676f6f676c654d425225323661642533443930253236707a|1\n d0bad0b0d0ba20d0bfd180d0bed0b4d0b0d0bcd0b820d0b4d0bbd18f20d0b4d0b5d0b2d183d188d0bad0b8|687474703a253246253246777777772e626f6e707269782e7275253235326625323532663737363925323532663131303931392d6c65766f652d676f6f676c652d7368746f72792e72752f666f72756d2f666f72756d2e6d617465722e72752f6461696c792f63616c63756c61746f72|1\n d0b6d0b0d180d0b5d0bdd18cd18f20d0b32ed181d183d180d0bed0b2d0b0d0bdd0b8d0b520d0b2d0bed180d0bed0bdd0b5d0b6d181d0bad0b0d18f20d0bed0b1d0bbd0b0d181d182d0bed0bfd180d0b8d0bbd0b520d0bfd0bed181d0bbd0b5d0b4d0bdd0b8d0b520d0bad0bed181d18b|687474703a2f2f756b7261696e627572672f65636f2d6d6c656b2f65636f6e646172792f73686f77746f7069632e7068703f69643d3436333837362e68746d6c3f69643d32303634313333363631253246676f6f676c652d4170706c655765624b69742532463533372e333620284b48544d4c2c206c696b65|1\n d180d0b8d0be20d0bdd0b020d0bad0b0d180d182d0bed187d0bdd0b8d186d0b020d181d0bcd0bed182d180d0b5d182d18c20d0bed0bdd0bbd0b0d0b9d0bd|687474703a2f2f73616d6172612e6972722e72752f636174616c6f675f676f6f676c654d425225323661642533443930253236707a|1.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_21() -> Result<()> { + let display = test_clickbench_query(21).await?; + assert_snapshot!(display, @""); + Ok(()) + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (1 total):\n d0bad0b0d0bad0bed0b920d0bfd0bbd0bed189d0b0d0b4d0bad0b8d0bcd0b820d0b4d0bed181d182d0b0d0b2d0bad0b8|687474703a253246253246766b2e636f6d2f696672616d652d6f77612e68746d6c3f313d31266369643d353737266f6b693d31266f705f63617465676f72795f69645d3d332673656c656374|d092d0b0d0bad0b0d0bdd181d0b8d18f20d091d0a0d090d09ad090d09d20d090d09dd094d0a0d095d0a1202d20d0bfd0bed0bfd0b0d0bbd0b820d0bad183d0bfd0b8d182d18c20d0b4d0bed0bcd0bed0b5d187d0bdd18bd0b520d188d0bad0b0d184d0b020476f6f676c652e636f6d203a3a20d0bad0bed182d182d0b5d0bad181d1822c20d091d183d180d18fd182d0bdd0b8d0bad0b820d0b4d0bbd18f20d0bfd0b5d187d18c20d0bcd0b5d0b1d0b5d0bbd18cd0b520d0b4d0bbd18f20d0b4d0b5d0b2d183d188d0bad0b0|5|1\n\nRows only in right (1 total):\n d0bad0bed0bfd182d0b8d0bcd0b8d0bad0b2d0b8d0b4d18b20d18ed180d0b8d0b920d0bfd0bed181d0bbd0b5d0b4d0bdd18fd18f|68747470733a2f2f70726f64756b747925324670756c6f76652e72752f626f6f6b6c79617474696f6e2d7761722d73696e696a2d393430343139342c3936323435332f666f746f|d09bd0b5d0b3d0bad0be20d0bdd0b020d183d187d0b0d181d182d0bdd18bd0b520d183d187d0b0d181d182d0bdd0b8d0bad0bed0b22e2c20d0a6d0b5d0bdd18b202d20d0a1d182d0b8d0bbd18cd0bdd0b0d18f20d0bfd0b0d180d0bdd0b5d0bc2e20d0a1d0b0d0b3d0b0d0bdd180d0bed0b320d0b4d0bed0b3d0b0d0b4d0b5d0bdd0b8d18f203a20d0a2d183d180d186d0b8d0b82c20d0bad183d0bfd0b8d182d18c20d18320313020d0b4d0bdd0b520d0bad0bed0bbd18cd0bdd18bd0b520d0bcd0b0d188d0b8d0bdd0bad0b820d0bdd0b520d0bfd180d0b5d0b4d181d182d0b0d0b2d0bad0b8202d20d09dd0bed0b2d0b0d18f20d18120d0b8d0b7d0b1d0b8d0b5d0bdd0b8d0b520d181d0bfd180d0bed0b4d0b0d0b6d0b03a20d0bad0bed182d18fd182d0b0203230313420d0b32ed0b22e20d0a6d0b5d0bdd0b03a2034373530302d313045434f30363020e28093202d2d2d2d2d2d2d2d20d0bad183d0bfd0b8d182d18c20d0bad0b2d0b0d180d182d0b8d180d18320d09ed180d0b5d0bdd0b1d183d180d0b32028d0a0d0bed181d181d0b8d0b82047616c616e7472617820466c616d696c6961646120476f6f676c652c204ed0be20313820d184d0bed182d0bed0bad0bed0bdd0b2d0b5d180d0ba20d0a1d183d0bfd0b5d18020d09ad0b0d180d0b4d0b8d0b3d0b0d0bd|5|1.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_22() -> Result<()> { + let display = test_clickbench_query(22).await?; + assert_snapshot!(display, @""); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_23() -> Result<()> { + let display = test_clickbench_query(23).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [EventTime@4 ASC NULLS LAST], fetch=10 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ SortExec: TopK(fetch=10), expr=[EventTime@4 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CAST(URL@13 AS Utf8View) LIKE %google% + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[WatchID, JavaEnable, Title, GoodEvent, EventTime, EventDate, CounterID, ClientIP, RegionID, UserID, CounterClass, OS, UserAgent, URL, Referer, IsRefresh, RefererCategoryID, RefererRegionID, URLCategoryID, URLRegionID, ResolutionWidth, ResolutionHeight, ResolutionDepth, FlashMajor, FlashMinor, FlashMinor2, NetMajor, NetMinor, UserAgentMajor, UserAgentMinor, CookieEnable, JavascriptEnable, IsMobile, MobilePhone, MobilePhoneModel, Params, IPNetworkID, TraficSourceID, SearchEngineID, SearchPhrase, AdvEngineID, IsArtifical, WindowClientWidth, WindowClientHeight, ClientTimeZone, ClientEventTime, SilverlightVersion1, SilverlightVersion2, SilverlightVersion3, SilverlightVersion4, PageCharset, CodeVersion, IsLink, IsDownload, IsNotBounce, FUniqID, OriginalURL, HID, IsOldCounter, IsEvent, IsParameter, DontCountHits, WithHash, HitColor, LocalEventTime, Age, Sex, Income, Interests, Robotness, RemoteIP, WindowName, OpenerName, HistoryLength, BrowserLanguage, BrowserCountry, SocialNetwork, SocialAction, HTTPError, SendTiming, DNSTiming, ConnectTiming, ResponseStartTiming, ResponseEndTiming, FetchTiming, SocialSourceNetworkID, SocialSourcePage, ParamPrice, ParamOrderID, ParamCurrency, ParamCurrencyID, OpenstatServiceName, OpenstatCampaignID, OpenstatAdID, OpenstatSourceID, UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm, FromTag, HasGCLID, RefererHash, URLHash, CLID], file_type=parquet, predicate=CAST(URL@13 AS Utf8View) LIKE %google% AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (1 total):\n d0b2d181d0bfd0bed0bcd0bdd0b8d182d18c20d181d0bed0bbd0bdd0b5d0bdd0b8d0b520d0b1d0b0d0bdd0bad0b020d0bbd0b0d0b420d184d0b8d0bbd18cd0bc\n\nRows only in right (1 total):\n d0bed182d0b2d0bed0b4d0b020d0b4d0bbd18f20d0bfd0b8d180d0bed0b6d0bad0b820d0bbd0b5d187d0b5d0bdd0bdd18b20d0b2d181d0b520d181d0b5d180d196d197.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_24() -> Result<()> { + let display = test_clickbench_query(24).await?; + assert_snapshot!(display, @""); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_25() -> Result<()> { + let display = test_clickbench_query(25).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [SearchPhrase@0 ASC NULLS LAST], fetch=10 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ SortExec: TopK(fetch=10), expr=[SearchPhrase@0 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: SearchPhrase@0 != + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchPhrase], file_type=parquet, predicate=SearchPhrase@0 != AND DynamicFilter [ empty ], pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_26() -> Result<()> { + let display = test_clickbench_query(26).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase] + │ SortPreservingMergeExec: [EventTime@1 ASC NULLS LAST, SearchPhrase@0 ASC NULLS LAST], fetch=10 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ SortExec: TopK(fetch=10), expr=[EventTime@1 ASC NULLS LAST, SearchPhrase@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[SearchPhrase@1 as SearchPhrase, EventTime@0 as EventTime] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: SearchPhrase@1 != + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventTime, SearchPhrase], file_type=parquet, predicate=SearchPhrase@1 != AND DynamicFilter [ empty ], pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_27() -> Result<()> { + let display = test_clickbench_query(27).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [l@1 DESC], fetch=25 + │ SortExec: TopK(fetch=25), expr=[l@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[CounterID@0 as CounterID, avg(length(hits.URL))@1 as l, count(Int64(1))@2 as c] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: count(Int64(1))@2 > 100000 + │ AggregateExec: mode=FinalPartitioned, gby=[CounterID@0 as CounterID], aggr=[avg(length(hits.URL)), count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CounterID@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[CounterID@0 as CounterID], aggr=[avg(length(hits.URL)), count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: URL@1 != + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[CounterID, URL], file_type=parquet, predicate=URL@1 != , pruning_predicate=URL_null_count@2 != row_count@3 AND (URL_min@0 != OR != URL_max@1), required_guarantees=[URL not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_28() -> Result<()> { + let display = test_clickbench_query(28).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [l@1 DESC], fetch=25 + │ SortExec: TopK(fetch=25), expr=[l@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0 as k, avg(length(hits.Referer))@1 as l, count(Int64(1))@2 as c, min(hits.Referer)@3 as min(hits.Referer)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: count(Int64(1))@2 > 100000 + │ AggregateExec: mode=FinalPartitioned, gby=[regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0 as regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))], aggr=[avg(length(hits.Referer)), count(Int64(1)), min(hits.Referer)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[regexp_replace(CAST(Referer@0 AS LargeUtf8), ^https?://(?:www\.)?([^/]+)/.*$, \1) as regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))], aggr=[avg(length(hits.Referer)), count(Int64(1)), min(hits.Referer)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: Referer@0 != + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[Referer], file_type=parquet, predicate=Referer@0 != , pruning_predicate=Referer_null_count@2 != row_count@3 AND (Referer_min@0 != OR != Referer_max@1), required_guarantees=[Referer not in ()] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_29() -> Result<()> { + let display = test_clickbench_query(29).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(hits.ResolutionWidth), sum(hits.ResolutionWidth + Int64(1)), sum(hits.ResolutionWidth + Int64(2)), sum(hits.ResolutionWidth + Int64(3)), sum(hits.ResolutionWidth + Int64(4)), sum(hits.ResolutionWidth + Int64(5)), sum(hits.ResolutionWidth + Int64(6)), sum(hits.ResolutionWidth + Int64(7)), sum(hits.ResolutionWidth + Int64(8)), sum(hits.ResolutionWidth + Int64(9)), sum(hits.ResolutionWidth + Int64(10)), sum(hits.ResolutionWidth + Int64(11)), sum(hits.ResolutionWidth + Int64(12)), sum(hits.ResolutionWidth + Int64(13)), sum(hits.ResolutionWidth + Int64(14)), sum(hits.ResolutionWidth + Int64(15)), sum(hits.ResolutionWidth + Int64(16)), sum(hits.ResolutionWidth + Int64(17)), sum(hits.ResolutionWidth + Int64(18)), sum(hits.ResolutionWidth + Int64(19)), sum(hits.ResolutionWidth + Int64(20)), sum(hits.ResolutionWidth + Int64(21)), sum(hits.ResolutionWidth + Int64(22)), sum(hits.ResolutionWidth + Int64(23)), sum(hits.ResolutionWidth + Int64(24)), sum(hits.ResolutionWidth + Int64(25)), sum(hits.ResolutionWidth + Int64(26)), sum(hits.ResolutionWidth + Int64(27)), sum(hits.ResolutionWidth + Int64(28)), sum(hits.ResolutionWidth + Int64(29)), sum(hits.ResolutionWidth + Int64(30)), sum(hits.ResolutionWidth + Int64(31)), sum(hits.ResolutionWidth + Int64(32)), sum(hits.ResolutionWidth + Int64(33)), sum(hits.ResolutionWidth + Int64(34)), sum(hits.ResolutionWidth + Int64(35)), sum(hits.ResolutionWidth + Int64(36)), sum(hits.ResolutionWidth + Int64(37)), sum(hits.ResolutionWidth + Int64(38)), sum(hits.ResolutionWidth + Int64(39)), sum(hits.ResolutionWidth + Int64(40)), sum(hits.ResolutionWidth + Int64(41)), sum(hits.ResolutionWidth + Int64(42)), sum(hits.ResolutionWidth + Int64(43)), sum(hits.ResolutionWidth + Int64(44)), sum(hits.ResolutionWidth + Int64(45)), sum(hits.ResolutionWidth + Int64(46)), sum(hits.ResolutionWidth + Int64(47)), sum(hits.ResolutionWidth + Int64(48)), sum(hits.ResolutionWidth + Int64(49)), sum(hits.ResolutionWidth + Int64(50)), sum(hits.ResolutionWidth + Int64(51)), sum(hits.ResolutionWidth + Int64(52)), sum(hits.ResolutionWidth + Int64(53)), sum(hits.ResolutionWidth + Int64(54)), sum(hits.ResolutionWidth + Int64(55)), sum(hits.ResolutionWidth + Int64(56)), sum(hits.ResolutionWidth + Int64(57)), sum(hits.ResolutionWidth + Int64(58)), sum(hits.ResolutionWidth + Int64(59)), sum(hits.ResolutionWidth + Int64(60)), sum(hits.ResolutionWidth + Int64(61)), sum(hits.ResolutionWidth + Int64(62)), sum(hits.ResolutionWidth + Int64(63)), sum(hits.ResolutionWidth + Int64(64)), sum(hits.ResolutionWidth + Int64(65)), sum(hits.ResolutionWidth + Int64(66)), sum(hits.ResolutionWidth + Int64(67)), sum(hits.ResolutionWidth + Int64(68)), sum(hits.ResolutionWidth + Int64(69)), sum(hits.ResolutionWidth + Int64(70)), sum(hits.ResolutionWidth + Int64(71)), sum(hits.ResolutionWidth + Int64(72)), sum(hits.ResolutionWidth + Int64(73)), sum(hits.ResolutionWidth + Int64(74)), sum(hits.ResolutionWidth + Int64(75)), sum(hits.ResolutionWidth + Int64(76)), sum(hits.ResolutionWidth + Int64(77)), sum(hits.ResolutionWidth + Int64(78)), sum(hits.ResolutionWidth + Int64(79)), sum(hits.ResolutionWidth + Int64(80)), sum(hits.ResolutionWidth + Int64(81)), sum(hits.ResolutionWidth + Int64(82)), sum(hits.ResolutionWidth + Int64(83)), sum(hits.ResolutionWidth + Int64(84)), sum(hits.ResolutionWidth + Int64(85)), sum(hits.ResolutionWidth + Int64(86)), sum(hits.ResolutionWidth + Int64(87)), sum(hits.ResolutionWidth + Int64(88)), sum(hits.ResolutionWidth + Int64(89))] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(hits.ResolutionWidth), sum(hits.ResolutionWidth + Int64(1)), sum(hits.ResolutionWidth + Int64(2)), sum(hits.ResolutionWidth + Int64(3)), sum(hits.ResolutionWidth + Int64(4)), sum(hits.ResolutionWidth + Int64(5)), sum(hits.ResolutionWidth + Int64(6)), sum(hits.ResolutionWidth + Int64(7)), sum(hits.ResolutionWidth + Int64(8)), sum(hits.ResolutionWidth + Int64(9)), sum(hits.ResolutionWidth + Int64(10)), sum(hits.ResolutionWidth + Int64(11)), sum(hits.ResolutionWidth + Int64(12)), sum(hits.ResolutionWidth + Int64(13)), sum(hits.ResolutionWidth + Int64(14)), sum(hits.ResolutionWidth + Int64(15)), sum(hits.ResolutionWidth + Int64(16)), sum(hits.ResolutionWidth + Int64(17)), sum(hits.ResolutionWidth + Int64(18)), sum(hits.ResolutionWidth + Int64(19)), sum(hits.ResolutionWidth + Int64(20)), sum(hits.ResolutionWidth + Int64(21)), sum(hits.ResolutionWidth + Int64(22)), sum(hits.ResolutionWidth + Int64(23)), sum(hits.ResolutionWidth + Int64(24)), sum(hits.ResolutionWidth + Int64(25)), sum(hits.ResolutionWidth + Int64(26)), sum(hits.ResolutionWidth + Int64(27)), sum(hits.ResolutionWidth + Int64(28)), sum(hits.ResolutionWidth + Int64(29)), sum(hits.ResolutionWidth + Int64(30)), sum(hits.ResolutionWidth + Int64(31)), sum(hits.ResolutionWidth + Int64(32)), sum(hits.ResolutionWidth + Int64(33)), sum(hits.ResolutionWidth + Int64(34)), sum(hits.ResolutionWidth + Int64(35)), sum(hits.ResolutionWidth + Int64(36)), sum(hits.ResolutionWidth + Int64(37)), sum(hits.ResolutionWidth + Int64(38)), sum(hits.ResolutionWidth + Int64(39)), sum(hits.ResolutionWidth + Int64(40)), sum(hits.ResolutionWidth + Int64(41)), sum(hits.ResolutionWidth + Int64(42)), sum(hits.ResolutionWidth + Int64(43)), sum(hits.ResolutionWidth + Int64(44)), sum(hits.ResolutionWidth + Int64(45)), sum(hits.ResolutionWidth + Int64(46)), sum(hits.ResolutionWidth + Int64(47)), sum(hits.ResolutionWidth + Int64(48)), sum(hits.ResolutionWidth + Int64(49)), sum(hits.ResolutionWidth + Int64(50)), sum(hits.ResolutionWidth + Int64(51)), sum(hits.ResolutionWidth + Int64(52)), sum(hits.ResolutionWidth + Int64(53)), sum(hits.ResolutionWidth + Int64(54)), sum(hits.ResolutionWidth + Int64(55)), sum(hits.ResolutionWidth + Int64(56)), sum(hits.ResolutionWidth + Int64(57)), sum(hits.ResolutionWidth + Int64(58)), sum(hits.ResolutionWidth + Int64(59)), sum(hits.ResolutionWidth + Int64(60)), sum(hits.ResolutionWidth + Int64(61)), sum(hits.ResolutionWidth + Int64(62)), sum(hits.ResolutionWidth + Int64(63)), sum(hits.ResolutionWidth + Int64(64)), sum(hits.ResolutionWidth + Int64(65)), sum(hits.ResolutionWidth + Int64(66)), sum(hits.ResolutionWidth + Int64(67)), sum(hits.ResolutionWidth + Int64(68)), sum(hits.ResolutionWidth + Int64(69)), sum(hits.ResolutionWidth + Int64(70)), sum(hits.ResolutionWidth + Int64(71)), sum(hits.ResolutionWidth + Int64(72)), sum(hits.ResolutionWidth + Int64(73)), sum(hits.ResolutionWidth + Int64(74)), sum(hits.ResolutionWidth + Int64(75)), sum(hits.ResolutionWidth + Int64(76)), sum(hits.ResolutionWidth + Int64(77)), sum(hits.ResolutionWidth + Int64(78)), sum(hits.ResolutionWidth + Int64(79)), sum(hits.ResolutionWidth + Int64(80)), sum(hits.ResolutionWidth + Int64(81)), sum(hits.ResolutionWidth + Int64(82)), sum(hits.ResolutionWidth + Int64(83)), sum(hits.ResolutionWidth + Int64(84)), sum(hits.ResolutionWidth + Int64(85)), sum(hits.ResolutionWidth + Int64(86)), sum(hits.ResolutionWidth + Int64(87)), sum(hits.ResolutionWidth + Int64(88)), sum(hits.ResolutionWidth + Int64(89))] + │ ProjectionExec: expr=[CAST(ResolutionWidth@0 AS Int64) as __common_expr_1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[ResolutionWidth], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_30() -> Result<()> { + let display = test_clickbench_query(30).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c@2 DESC], fetch=10 + │ SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[SearchEngineID@0 as SearchEngineID, ClientIP@1 as ClientIP, count(Int64(1))@2 as c, sum(hits.IsRefresh)@3 as sum(hits.IsRefresh), avg(hits.ResolutionWidth)@4 as avg(hits.ResolutionWidth)] + │ AggregateExec: mode=FinalPartitioned, gby=[SearchEngineID@0 as SearchEngineID, ClientIP@1 as ClientIP], aggr=[count(Int64(1)), sum(hits.IsRefresh), avg(hits.ResolutionWidth)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([SearchEngineID@0, ClientIP@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[SearchEngineID@3 as SearchEngineID, ClientIP@0 as ClientIP], aggr=[count(Int64(1)), sum(hits.IsRefresh), avg(hits.ResolutionWidth)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: SearchPhrase@4 != , projection=[ClientIP@0, IsRefresh@1, ResolutionWidth@2, SearchEngineID@3] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[ClientIP, IsRefresh, ResolutionWidth, SearchEngineID, SearchPhrase], file_type=parquet, predicate=SearchPhrase@4 != , pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (10 total):\n 8673025726158767406|1264438551|1|0|1990.00\n 5320052218057629211|-1703087277|1|0|1996.00\n 6244273852606083750|1554672832|1|0|1638.00\n 8628753750962053665|1215278356|1|0|1087.00\n 7035318163404387241|1326714320|1|0|1638.00\n 8431857775494210873|1237512945|1|0|1996.00\n 5110752526539992124|37611695|1|0|1917.00\n 8986794334343068049|1860752926|1|0|1638.00\n 8044147848299485837|1382122372|1|0|1368.00\n 7936057634954670727|1897481896|1|0|1638.00\n\nRows only in right (10 total):\n 5132615111782210132|-50313020|1|0|1368.00\n 5783789691451717551|-1310327384|1|0|1638.00\n 5756260993772351383|1484317883|1|0|375.00\n 7739310142000732364|991864113|1|0|1368.00\n 7593472904893539271|-151291403|1|0|1087.00\n 6339599967989898410|1543815587|1|0|1638.00\n 7794346560421945218|1645556180|1|0|1368.00\n 6112645108657361792|593586188|1|0|1638.00\n 6675910710751922756|-816379256|1|0|1368.00\n 5802727636196431835|1986422271|1|0|1996.00.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_31() -> Result<()> { + let display = test_clickbench_query(31).await?; + assert_snapshot!(display, @""); + Ok(()) + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (10 total):\n 7643059318918524417|1767085700|1|0|0.00\n 5437163248266133938|-1465369615|1|0|0.00\n 9142541582422390102|-1465369615|1|0|0.00\n 8438994503411842126|-1465369615|1|0|0.00\n 7362096505818029859|-565678477|1|0|0.00\n 4928022308880516715|1699955284|1|0|0.00\n 5269769817689282522|1699955284|1|0|0.00\n 9081648050908046886|1699955284|1|0|0.00\n 6824181869275536503|1699955284|1|0|0.00\n 6905712404475757487|1552811156|1|0|0.00\n\nRows only in right (10 total):\n 6967277596165459879|-941091661|1|0|1368.00\n 5796237228224217668|-1310327384|1|0|1638.00\n 7218628137278606666|1511490240|1|0|1638.00\n 8314760197723815280|1566105210|1|0|1996.00\n 7053263954762394007|757778490|1|0|339.00\n 6283334114093174531|1216031795|1|0|1368.00\n 8818295356247036741|83042182|1|0|1638.00\n 6620528864937282562|-862894777|1|0|1996.00\n 8466121050002905379|83042182|1|0|1638.00\n 7554844936512227411|-1746904856|1|0|1368.00.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_32() -> Result<()> { + let display = test_clickbench_query(32).await?; + assert_snapshot!(display, @""); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_33() -> Result<()> { + let display = test_clickbench_query(33).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c@1 DESC], fetch=10 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[c@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[URL@0 as URL, count(Int64(1))@1 as c] + │ AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([URL@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[URL], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_34() -> Result<()> { + let display = test_clickbench_query(34).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c@2 DESC], fetch=10 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[1 as Int64(1), URL@0 as URL, count(Int64(1))@1 as c] + │ AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([URL@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[URL], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_35() -> Result<()> { + let display = test_clickbench_query(35).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c@4 DESC], fetch=10 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[c@4 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[ClientIP@0 as ClientIP, hits.ClientIP - Int64(1)@1 as hits.ClientIP - Int64(1), hits.ClientIP - Int64(2)@2 as hits.ClientIP - Int64(2), hits.ClientIP - Int64(3)@3 as hits.ClientIP - Int64(3), count(Int64(1))@4 as c] + │ AggregateExec: mode=FinalPartitioned, gby=[ClientIP@0 as ClientIP, hits.ClientIP - Int64(1)@1 as hits.ClientIP - Int64(1), hits.ClientIP - Int64(2)@2 as hits.ClientIP - Int64(2), hits.ClientIP - Int64(3)@3 as hits.ClientIP - Int64(3)], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ClientIP@0, hits.ClientIP - Int64(1)@1, hits.ClientIP - Int64(2)@2, hits.ClientIP - Int64(3)@3], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ClientIP@1 as ClientIP, __common_expr_1@0 - 1 as hits.ClientIP - Int64(1), __common_expr_1@0 - 2 as hits.ClientIP - Int64(2), __common_expr_1@0 - 3 as hits.ClientIP - Int64(3)], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[CAST(ClientIP@0 AS Int64) as __common_expr_1, ClientIP@0 as ClientIP] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[ClientIP], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_36() -> Result<()> { + let display = test_clickbench_query(36).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [pageviews@1 DESC], fetch=10 + │ SortExec: TopK(fetch=10), expr=[pageviews@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[URL@0 as URL, count(Int64(1))@1 as pageviews] + │ AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([URL@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND DontCountHits@4 = 0 AND IsRefresh@3 = 0 AND URL@2 != , projection=[URL@2] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventDate, CounterID, URL, IsRefresh, DontCountHits], file_type=parquet, predicate=CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND DontCountHits@4 = 0 AND IsRefresh@3 = 0 AND URL@2 != , pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND DontCountHits_null_count@6 != row_count@3 AND DontCountHits_min@4 <= 0 AND 0 <= DontCountHits_max@5 AND IsRefresh_null_count@9 != row_count@3 AND IsRefresh_min@7 <= 0 AND 0 <= IsRefresh_max@8 AND URL_null_count@12 != row_count@3 AND (URL_min@10 != OR != URL_max@11), required_guarantees=[CounterID in (62), DontCountHits in (0), IsRefresh in (0), URL not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_37() -> Result<()> { + let display = test_clickbench_query(37).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [pageviews@1 DESC], fetch=10 + │ SortExec: TopK(fetch=10), expr=[pageviews@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[Title@0 as Title, count(Int64(1))@1 as pageviews] + │ AggregateExec: mode=FinalPartitioned, gby=[Title@0 as Title], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([Title@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[Title@0 as Title], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CounterID@2 = 62 AND CAST(EventDate@1 AS Utf8) >= 2013-07-01 AND CAST(EventDate@1 AS Utf8) <= 2013-07-31 AND DontCountHits@4 = 0 AND IsRefresh@3 = 0 AND Title@0 != , projection=[Title@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[Title, EventDate, CounterID, IsRefresh, DontCountHits], file_type=parquet, predicate=CounterID@2 = 62 AND CAST(EventDate@1 AS Utf8) >= 2013-07-01 AND CAST(EventDate@1 AS Utf8) <= 2013-07-31 AND DontCountHits@4 = 0 AND IsRefresh@3 = 0 AND Title@0 != , pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND DontCountHits_null_count@6 != row_count@3 AND DontCountHits_min@4 <= 0 AND 0 <= DontCountHits_max@5 AND IsRefresh_null_count@9 != row_count@3 AND IsRefresh_min@7 <= 0 AND 0 <= IsRefresh_max@8 AND Title_null_count@12 != row_count@3 AND (Title_min@10 != OR != Title_max@11), required_guarantees=[CounterID in (62), DontCountHits in (0), IsRefresh in (0), Title not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_38() -> Result<()> { + let display = test_clickbench_query(38).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ GlobalLimitExec: skip=1000, fetch=10 + │ SortPreservingMergeExec: [pageviews@1 DESC], fetch=1010 + │ SortExec: TopK(fetch=1010), expr=[pageviews@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[URL@0 as URL, count(Int64(1))@1 as pageviews] + │ AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([URL@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@3 = 0 AND IsLink@4 != 0 AND IsDownload@5 = 0, projection=[URL@2] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventDate, CounterID, URL, IsRefresh, IsLink, IsDownload], file_type=parquet, predicate=CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@3 = 0 AND IsLink@4 != 0 AND IsDownload@5 = 0, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5 AND IsLink_null_count@9 != row_count@3 AND (IsLink_min@7 != 0 OR 0 != IsLink_max@8) AND IsDownload_null_count@12 != row_count@3 AND IsDownload_min@10 <= 0 AND 0 <= IsDownload_max@11, required_guarantees=[CounterID in (62), IsDownload in (0), IsLink not in (0), IsRefresh in (0)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_39() -> Result<()> { + let display = test_clickbench_query(39).await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ GlobalLimitExec: skip=1000, fetch=10 + │ SortPreservingMergeExec: [pageviews@5 DESC], fetch=1010 + │ SortExec: TopK(fetch=1010), expr=[pageviews@5 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[TraficSourceID@0 as TraficSourceID, SearchEngineID@1 as SearchEngineID, AdvEngineID@2 as AdvEngineID, CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END@3 as src, URL@4 as dst, count(Int64(1))@5 as pageviews] + │ AggregateExec: mode=FinalPartitioned, gby=[TraficSourceID@0 as TraficSourceID, SearchEngineID@1 as SearchEngineID, AdvEngineID@2 as AdvEngineID, CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END@3 as CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END, URL@4 as URL], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([TraficSourceID@0, SearchEngineID@1, AdvEngineID@2, CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END@3, URL@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[TraficSourceID@2 as TraficSourceID, SearchEngineID@3 as SearchEngineID, AdvEngineID@4 as AdvEngineID, CASE WHEN SearchEngineID@3 = 0 AND AdvEngineID@4 = 0 THEN Referer@1 ELSE END as CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END, URL@0 as URL], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@4 = 0, projection=[URL@2, Referer@3, TraficSourceID@5, SearchEngineID@6, AdvEngineID@7] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventDate, CounterID, URL, Referer, IsRefresh, TraficSourceID, SearchEngineID, AdvEngineID], file_type=parquet, predicate=CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@4 = 0, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5, required_guarantees=[CounterID in (62), IsRefresh in (0)] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_40() -> Result<()> { + let display = test_clickbench_query(40).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ GlobalLimitExec: skip=100, fetch=10 + │ SortPreservingMergeExec: [pageviews@2 DESC], fetch=110 + │ SortExec: TopK(fetch=110), expr=[pageviews@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[URLHash@0 as URLHash, EventDate@1 as EventDate, count(Int64(1))@2 as pageviews] + │ AggregateExec: mode=FinalPartitioned, gby=[URLHash@0 as URLHash, EventDate@1 as EventDate], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([URLHash@0, EventDate@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[URLHash@1 as URLHash, EventDate@0 as EventDate], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@2 = 0 AND (TraficSourceID@3 = -1 OR TraficSourceID@3 = 6) AND RefererHash@4 = 3594120000172545465, projection=[EventDate@0, URLHash@5] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventDate, CounterID, IsRefresh, TraficSourceID, RefererHash, URLHash], file_type=parquet, predicate=CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@2 = 0 AND (TraficSourceID@3 = -1 OR TraficSourceID@3 = 6) AND RefererHash@4 = 3594120000172545465, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5 AND (TraficSourceID_null_count@9 != row_count@3 AND TraficSourceID_min@7 <= -1 AND -1 <= TraficSourceID_max@8 OR TraficSourceID_null_count@9 != row_count@3 AND TraficSourceID_min@7 <= 6 AND 6 <= TraficSourceID_max@8) AND RefererHash_null_count@12 != row_count@3 AND RefererHash_min@10 <= 3594120000172545465 AND 3594120000172545465 <= RefererHash_max@11, required_guarantees=[CounterID in (62), IsRefresh in (0), RefererHash in (3594120000172545465), TraficSourceID in (-1, 6)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_41() -> Result<()> { + let display = test_clickbench_query(41).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ GlobalLimitExec: skip=10000, fetch=10 + │ SortPreservingMergeExec: [pageviews@2 DESC], fetch=10010 + │ SortExec: TopK(fetch=10010), expr=[pageviews@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[WindowClientWidth@0 as WindowClientWidth, WindowClientHeight@1 as WindowClientHeight, count(Int64(1))@2 as pageviews] + │ AggregateExec: mode=FinalPartitioned, gby=[WindowClientWidth@0 as WindowClientWidth, WindowClientHeight@1 as WindowClientHeight], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([WindowClientWidth@0, WindowClientHeight@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[WindowClientWidth@0 as WindowClientWidth, WindowClientHeight@1 as WindowClientHeight], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@2 = 0 AND DontCountHits@5 = 0 AND URLHash@6 = 2868770270353813622, projection=[WindowClientWidth@3, WindowClientHeight@4] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventDate, CounterID, IsRefresh, WindowClientWidth, WindowClientHeight, DontCountHits, URLHash], file_type=parquet, predicate=CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@2 = 0 AND DontCountHits@5 = 0 AND URLHash@6 = 2868770270353813622, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5 AND DontCountHits_null_count@9 != row_count@3 AND DontCountHits_min@7 <= 0 AND 0 <= DontCountHits_max@8 AND URLHash_null_count@12 != row_count@3 AND URLHash_min@10 <= 2868770270353813622 AND 2868770270353813622 <= URLHash_max@11, required_guarantees=[CounterID in (62), DontCountHits in (0), IsRefresh in (0), URLHash in (2868770270353813622)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + #[ignore = "ordering mismatch: expected ordering: Some(LexOrdering { exprs: [PhysicalSortExpr { expr: ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }, options: SortOptions { descending: false, nulls_first: false } }], set: {ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }} }), actual ordering: Some(LexOrdering { exprs: [PhysicalSortExpr { expr: ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }, options: SortOptions { descending: false, nulls_first: false } }], set: {ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }} })"] + async fn test_clickbench_42() -> Result<()> { + let display = test_clickbench_query(42).await?; + assert_snapshot!(display, @""); + Ok(()) + } + + static INIT_TEST_TPCDS_TABLES: OnceCell<()> = OnceCell::const_new(); + + async fn test_clickbench_query(query_id: usize) -> Result { + let data_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(format!( + "testdata/clickbench/plans_range{}-{}", + FILE_RANGE.start, FILE_RANGE.end + )); + INIT_TEST_TPCDS_TABLES + .get_or_init(|| async { + clickbench::generate_clickbench_data(&data_dir, FILE_RANGE) + .await + .unwrap(); + }) + .await; + + let query_sql = clickbench::get_test_clickbench_query(query_id)?; + + let (d_ctx, _guard) = start_localhost_context(NUM_WORKERS, DefaultSessionBuilder).await; + let d_ctx = d_ctx + .with_distributed_files_per_task(FILES_PER_TASK)? + .with_distributed_cardinality_effect_task_scale_factor(CARDINALITY_TASK_COUNT_FACTOR)?; + + clickbench::register_tables(&d_ctx, &data_dir).await?; + + let df = d_ctx.sql(&query_sql).await?; + let plan = df.create_physical_plan().await?; + + if !plan.as_any().is::() { + Ok("".to_string()) + } else { + Ok(display_plan_ascii(plan.as_ref(), false)) + } + } +} From a94556a4732ca13dec4a2d13dbb074b451d254c5 Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Mon, 29 Dec 2025 14:14:30 +0100 Subject: [PATCH 07/27] Distribute UNION operations (#262) * Split channel resolver in two * Simplify WorkerResolverExtension and ChannelResolverExtension * Add default builder to ArrowFlightEndpoint * Add some docs * Listen to clippy * Split get_flight_client_for_url in two * Fix conflicts * Remove unnecessary channel resolver * Improve WorkerResolver docs * Use one ChannelResolver per runtime * Improve error reporting on client connection failure * Add a from_session_builder method for constructing an InMemoryChannelResolver * Add ChannelResolver and WorkerResolver default implementations for Arcs * Make TPC-DS tests use DataFusion test dataset * Remove non-working in-memory option from benchmarks * Remove unnecessary utils folder * Refactor benchmark folder * Rename to prepare_tpch.rs * Adapt benchmarks for TPC-DS * Update benchmarks README.md * Fix conflicts * Use default session state builder * Add ChildrenIsolatorUnionExec * Add proto serde for ChildrenIsolatorUnionExec * Wire up ChildrenIsolatorUnionExec to planner * Add integration tests for distributed UNIONs * Skip query 72 in TPC-DS benchmarks * Allow setting children isolator unions * Allow passing multiple queries in benchmarks --- benchmarks/get-tpcds.sh | 21 + benchmarks/src/run.rs | 15 +- src/distributed_ext.rs | 57 + src/distributed_planner/distributed_config.rs | 8 + .../distributed_physical_optimizer_rule.rs | 159 + src/distributed_planner/plan_annotator.rs | 143 +- .../children_isolator_union.rs | 541 +++ src/execution_plans/mod.rs | 2 + src/protobuf/distributed_codec.rs | 125 +- src/stage.rs | 2 +- tests/distributed_unions.rs | 211 ++ tests/tpcds_plans_test.rs | 3127 +++++++++-------- 12 files changed, 2828 insertions(+), 1583 deletions(-) create mode 100644 benchmarks/get-tpcds.sh create mode 100644 src/execution_plans/children_isolator_union.rs create mode 100644 tests/distributed_unions.rs diff --git a/benchmarks/get-tpcds.sh b/benchmarks/get-tpcds.sh new file mode 100644 index 00000000..3a41b40b --- /dev/null +++ b/benchmarks/get-tpcds.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -e + +SCALE_FACTOR=${SCALE_FACTOR:-1} +PARTITIONS=${PARTITIONS:-16} + +echo "Generating TPC-DS dataset with SCALE_FACTOR=${SCALE_FACTOR} and PARTITIONS=${PARTITIONS}" + +# https://stackoverflow.com/questions/59895/how-do-i-get-the-directory-where-a-bash-script-is-located-from-within-the-script +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +DATA_DIR=${DATA_DIR:-$SCRIPT_DIR/data} +CARGO_COMMAND=${CARGO_COMMAND:-"cargo run --release"} +TPCDS_DIR="${DATA_DIR}/tpcds_sf${SCALE_FACTOR}" + +echo "Creating tpcds dataset at Scale Factor ${SCALE_FACTOR} in ${TPCDS_DIR}..." + +# Ensure the target data directory exists +mkdir -p "${TPCDS_DIR}" + +$CARGO_COMMAND -- prepare-tpcds --output "${TPCDS_DIR}" --partitions "$PARTITIONS" diff --git a/benchmarks/src/run.rs b/benchmarks/src/run.rs index 52900756..60b91dae 100644 --- a/benchmarks/src/run.rs +++ b/benchmarks/src/run.rs @@ -58,8 +58,8 @@ use tonic::transport::Server; #[structopt(verbatim_doc_comment)] pub struct RunOpt { /// Query number. If not specified, runs all queries - #[structopt(short, long)] - pub query: Option, + #[structopt(short, long, use_delimiter = true)] + pub query: Vec, /// Path to data files #[structopt(parse(from_os_str), short = "p", long = "path")] @@ -89,6 +89,10 @@ pub struct RunOpt { #[structopt(long)] cardinality_task_sf: Option, + /// Use children isolator UNIONs for distributing UNION operations. + #[structopt(long)] + children_isolator_unions: bool, + /// Collects metrics across network boundaries #[structopt(long)] collect_metrics: bool, @@ -143,7 +147,9 @@ impl Dataset { Dataset::Tpch => (1..22 + 1) .map(|i| Ok((i as usize, tpch::get_test_tpch_query(i)?))) .collect(), - Dataset::Tpcds => (1..99 + 1) + Dataset::Tpcds => (1..72) + // skip query 72, it's ridiculously slow + .chain(73..99 + 1) .map(|i| Ok((i, tpcds::get_test_tpcds_query(i)?))) .collect(), Dataset::Clickbench => (0..42 + 1) @@ -203,6 +209,7 @@ impl RunOpt { .with_distributed_cardinality_effect_task_scale_factor( self.cardinality_task_sf.unwrap_or(1.0), )? + .with_distributed_children_isolator_unions(self.children_isolator_unions)? .with_distributed_metrics_collection(self.collect_metrics)? .build(); let ctx = SessionContext::new_with_state(state); @@ -219,7 +226,7 @@ impl RunOpt { let dataset = Dataset::infer_from_data_path(path.clone())?; for (id, sql) in dataset.queries()? { - if self.query.is_some_and(|v| v != id) { + if !self.query.is_empty() && !self.query.contains(&id) { continue; } let query_id = format!("{dataset:?} {id}"); diff --git a/src/distributed_ext.rs b/src/distributed_ext.rs index 95b20f6c..4228f4cf 100644 --- a/src/distributed_ext.rs +++ b/src/distributed_ext.rs @@ -418,6 +418,35 @@ pub trait DistributedExt: Sized { /// Same as [DistributedExt::with_distributed_metrics_collection] but with an in-place mutation. fn set_distributed_metrics_collection(&mut self, enabled: bool) -> Result<(), DataFusionError>; + + /// Enables children isolator unions for distributing UNION operations across as many tasks as + /// the sum of all the tasks required for each child. + /// + /// For example, if there is a UNION with 3 children, requiring one task each, it will result + /// in a plan with 3 tasks where each task runs one child: + /// + /// ```text + /// ┌─────────────────────────────┐┌─────────────────────────────┐┌─────────────────────────────┐ + /// │ Task 1 ││ Task 2 ││ Task 3 │ + /// │┌───────────────────────────┐││┌───────────────────────────┐││┌───────────────────────────┐│ + /// ││ ChildrenIsolatorUnionExec ││││ ChildrenIsolatorUnionExec ││││ ChildrenIsolatorUnionExec ││ + /// │└───▲─────────▲─────────▲───┘││└───▲─────────▲─────────▲───┘││└───▲─────────▲─────────▲───┘│ + /// │ │ ││ │ ││ │ │ + /// │┌───┴───┐ ┌ ─│ ─ ┌ ─│ ─ ││┌ ─│ ─ ┌───┴───┐ ┌ ─│ ─ ││┌ ─│ ─ ┌ ─│ ─ ┌───┴───┐│ + /// ││Child 1│ Child 2│ Child 3│││ Child 1│ │Child 2│ Child 3│││ Child 1│ Child 2│ │Child 3││ + /// │└───────┘ └ ─ ─ └ ─ ─ ││└ ─ ─ └───────┘ └ ─ ─ ││└ ─ ─ └ ─ ─ └───────┘│ + /// └─────────────────────────────┘└─────────────────────────────┘└─────────────────────────────┘ + /// ``` + fn with_distributed_children_isolator_unions( + self, + enabled: bool, + ) -> Result; + + /// Same as [DistributedExt::with_distributed_children_isolator_unions] but with an in-place mutation. + fn set_distributed_children_isolator_unions( + &mut self, + enabled: bool, + ) -> Result<(), DataFusionError>; } impl DistributedExt for SessionConfig { @@ -489,6 +518,15 @@ impl DistributedExt for SessionConfig { Ok(()) } + fn set_distributed_children_isolator_unions( + &mut self, + enabled: bool, + ) -> Result<(), DataFusionError> { + let d_cfg = DistributedConfig::from_config_options_mut(self.options_mut())?; + d_cfg.children_isolator_unions = enabled; + Ok(()) + } + delegate! { to self { #[call(set_distributed_option_extension)] @@ -530,6 +568,10 @@ impl DistributedExt for SessionConfig { #[call(set_distributed_metrics_collection)] #[expr($?;Ok(self))] fn with_distributed_metrics_collection(mut self, enabled: bool) -> Result; + + #[call(set_distributed_children_isolator_unions)] + #[expr($?;Ok(self))] + fn with_distributed_children_isolator_unions(mut self, enabled: bool) -> Result; } } } @@ -586,6 +628,11 @@ impl DistributedExt for SessionStateBuilder { #[call(set_distributed_metrics_collection)] #[expr($?;Ok(self))] fn with_distributed_metrics_collection(mut self, enabled: bool) -> Result; + + fn set_distributed_children_isolator_unions(&mut self, enabled: bool) -> Result<(), DataFusionError>; + #[call(set_distributed_children_isolator_unions)] + #[expr($?;Ok(self))] + fn with_distributed_children_isolator_unions(mut self, enabled: bool) -> Result; } } } @@ -642,6 +689,11 @@ impl DistributedExt for SessionState { #[call(set_distributed_metrics_collection)] #[expr($?;Ok(self))] fn with_distributed_metrics_collection(mut self, enabled: bool) -> Result; + + fn set_distributed_children_isolator_unions(&mut self, enabled: bool) -> Result<(), DataFusionError>; + #[call(set_distributed_children_isolator_unions)] + #[expr($?;Ok(self))] + fn with_distributed_children_isolator_unions(mut self, enabled: bool) -> Result; } } } @@ -698,6 +750,11 @@ impl DistributedExt for SessionContext { #[call(set_distributed_metrics_collection)] #[expr($?;Ok(self))] fn with_distributed_metrics_collection(self, enabled: bool) -> Result; + + fn set_distributed_children_isolator_unions(&mut self, enabled: bool) -> Result<(), DataFusionError>; + #[call(set_distributed_children_isolator_unions)] + #[expr($?;Ok(self))] + fn with_distributed_children_isolator_unions(self, enabled: bool) -> Result; } } } diff --git a/src/distributed_planner/distributed_config.rs b/src/distributed_planner/distributed_config.rs index c85b8e6e..5eaea74a 100644 --- a/src/distributed_planner/distributed_config.rs +++ b/src/distributed_planner/distributed_config.rs @@ -28,6 +28,14 @@ extensions_options! { /// batches over the wire. /// If set to 0, batch coalescing is disabled on network shuffle operations. pub shuffle_batch_size: usize, default = 8192 + /// When encountering a UNION operation, isolate its children depending on the task context. + /// For example, on a UNION operation with 3 children running in 3 distributed tasks, + /// instead of executing the 3 children in each 3 tasks with a DistributedTaskContext of + /// 1/3, 2/3, and 3/3 respectively, Execute: + /// - The first child in the first task with a DistributedTaskContext of 1/1 + /// - The second child in the second task with a DistributedTaskContext of 1/1 + /// - The third child in the third task with a DistributedTaskContext of 1/1 + pub children_isolator_unions: bool, default = true /// Propagate collected metrics from all nodes in the plan across network boundaries /// so that they can be reconstructed on the head node of the plan. pub collect_metrics: bool, default = true diff --git a/src/distributed_planner/distributed_physical_optimizer_rule.rs b/src/distributed_planner/distributed_physical_optimizer_rule.rs index 2f78b83a..c3e4e79d 100644 --- a/src/distributed_planner/distributed_physical_optimizer_rule.rs +++ b/src/distributed_planner/distributed_physical_optimizer_rule.rs @@ -601,6 +601,165 @@ mod tests { "); } + #[tokio::test] + async fn test_unioning_2_tables() { + let query = r#" + set distributed.children_isolator_unions=true; + SELECT "MinTemp", "RainToday" FROM weather WHERE "MinTemp" > 10.0 + UNION ALL + SELECT "MaxTemp", "RainToday" FROM weather WHERE "MaxTemp" < 30.0 + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(6)) + }) + .await; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=24, input_tasks=6 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] t4:[p16..p19] t5:[p20..p23] + │ DistributedUnionExec: t0:[c0(0/3)] t1:[c0(1/3)] t2:[c0(2/3)] t3:[c1(0/3)] t4:[c1(1/3)] t5:[c1(2/3)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: MinTemp@0 > 10 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: MaxTemp@0 < 30 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@0 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + └────────────────────────────────────────────────── + "); + } + + #[tokio::test] + async fn test_unioning_2_tables_limited_workers() { + let query = r#" + set distributed.children_isolator_unions=true; + SELECT "MinTemp", "RainToday" FROM weather WHERE "MinTemp" > 10.0 + UNION ALL + SELECT "MaxTemp", "RainToday" FROM weather WHERE "MaxTemp" < 30.0 + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) + }) + .await; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=12, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] + │ DistributedUnionExec: t0:[c0] t1:[c1(0/2)] t2:[c1(1/2)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: MinTemp@0 > 10 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: MaxTemp@0 < 30 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__] t1:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@0 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + └────────────────────────────────────────────────── + "); + } + + #[tokio::test] + async fn test_unioning_3_tables() { + let query = r#" + set distributed.children_isolator_unions=true; + SELECT "MinTemp", "RainToday" FROM weather WHERE "MinTemp" > 10.0 + UNION ALL + SELECT "MaxTemp", "RainToday" FROM weather WHERE "MaxTemp" < 30.0 + UNION ALL + SELECT "Temp9am", "RainToday" FROM weather WHERE "Temp9am" > 15.0 + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) + }) + .await; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=12, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: MinTemp@0 > 10 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: MaxTemp@0 < 30 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@0 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: Temp9am@0 > 15 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@0 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] + └────────────────────────────────────────────────── + "); + } + + #[tokio::test] + async fn test_unioning_5_tables() { + let query = r#" + set distributed.children_isolator_unions=true; + SELECT "MinTemp", "RainToday" FROM weather WHERE "MinTemp" > 10.0 + UNION ALL + SELECT "MaxTemp", "RainToday" FROM weather WHERE "MaxTemp" < 30.0 + UNION ALL + SELECT "Temp9am", "RainToday" FROM weather WHERE "Temp9am" > 15.0 + UNION ALL + SELECT "Temp3pm", "RainToday" FROM weather WHERE "Temp3pm" < 25.0 + UNION ALL + SELECT "Rainfall", "RainToday" FROM weather WHERE "Rainfall" > 5.0 + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) + }) + .await; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=24, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p7] t1:[p8..p15] t2:[p16..p23] + │ DistributedUnionExec: t0:[c0, c1] t1:[c2, c3] t2:[c4] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: MinTemp@0 > 10 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: MaxTemp@0 < 30 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@0 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: Temp9am@0 > 15 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@0 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] + │ ProjectionExec: expr=[Temp3pm@0 as MinTemp, RainToday@1 as RainToday] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: Temp3pm@0 < 25 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[Temp3pm, RainToday], file_type=parquet, predicate=Temp3pm@0 < 25, pruning_predicate=Temp3pm_null_count@1 != row_count@2 AND Temp3pm_min@0 < 25, required_guarantees=[] + │ ProjectionExec: expr=[Rainfall@0 as MinTemp, RainToday@1 as RainToday] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: Rainfall@0 > 5 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=Rainfall@0 > 5, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 5, required_guarantees=[] + └────────────────────────────────────────────────── + "); + } + async fn sql_to_explain( query: &str, f: impl FnOnce(SessionStateBuilder) -> SessionStateBuilder, diff --git a/src/distributed_planner/plan_annotator.rs b/src/distributed_planner/plan_annotator.rs index 10cae1ca..9704f646 100644 --- a/src/distributed_planner/plan_annotator.rs +++ b/src/distributed_planner/plan_annotator.rs @@ -1,3 +1,4 @@ +use crate::execution_plans::ChildrenIsolatorUnionExec; use crate::{DistributedConfig, TaskCountAnnotation, TaskEstimator}; use datafusion::common::{DataFusionError, plan_datafusion_err}; use datafusion::config::ConfigOptions; @@ -8,6 +9,7 @@ use datafusion::physical_plan::execution_plan::CardinalityEffect; use datafusion::physical_plan::joins::{HashJoinExec, PartitionMode}; use datafusion::physical_plan::repartition::RepartitionExec; use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; +use datafusion::physical_plan::union::UnionExec; use std::fmt::{Debug, Formatter}; use std::sync::Arc; @@ -168,30 +170,40 @@ fn _annotate_plan( } } - // The task count for this plan is decided by the biggest task count from the children; unless - // a child specifies a maximum task count, in that case, the maximum is respected. Some - // nodes can only run in one task. If there is a subplan with a single node declaring that - // it can only run in one task, all the rest of the nodes in the stage need to respect it. let mut task_count = Desired(1); - for annotated_child in annotated_children.iter() { - task_count = match (task_count, &annotated_child.task_count) { - (Desired(desired), Desired(child)) => Desired(desired.max(*child)), - (Maximum(max), Desired(_)) => Maximum(max), - (Desired(_), Maximum(max)) => Maximum(*max), - (Maximum(max_1), Maximum(max_2)) => Maximum(max_1.min(*max_2)), - }; - task_count = task_count.limit(n_workers); - } - - // We cannot distribute CollectLeft HashJoinExec nodes yet. Once - // https://github.com/datafusion-contrib/datafusion-distributed/pull/229 lands, - // we can remove this check. - if let Some(node) = plan.as_any().downcast_ref::() { - if node.mode == PartitionMode::CollectLeft { - task_count = Maximum(1); + if d_cfg.children_isolator_unions && plan.as_any().is::() { + // Unions have the chance to decide how many tasks they should run on. If there's a union + // with a bunch of children, the user might want to increase parallelism and increase the + // task count for the stage running that. + let mut count = 0; + for annotated_child in annotated_children.iter() { + count += annotated_child.task_count.as_usize(); + } + task_count = Desired(count); + } else if let Some(node) = plan.as_any().downcast_ref::() + && node.mode == PartitionMode::CollectLeft + { + // We cannot distribute CollectLeft HashJoinExec nodes yet. Once + // https://github.com/datafusion-contrib/datafusion-distributed/pull/229 lands, + // we can remove this check. + task_count = Maximum(1); + } else { + // The task count for this plan is decided by the biggest task count from the children; unless + // a child specifies a maximum task count, in that case, the maximum is respected. Some + // nodes can only run in one task. If there is a subplan with a single node declaring that + // it can only run in one task, all the rest of the nodes in the stage need to respect it. + for annotated_child in annotated_children.iter() { + task_count = match (task_count, &annotated_child.task_count) { + (Desired(desired), Desired(child)) => Desired(desired.max(*child)), + (Maximum(max), Desired(_)) => Maximum(max), + (Desired(_), Maximum(max)) => Maximum(*max), + (Maximum(max_1), Maximum(max_2)) => Maximum(max_1.min(*max_2)), + }; } } + task_count = task_count.limit(n_workers); + // The plan does not need a NetworkBoundary, so just take the biggest task count from // the children and annotate the plan with that. let mut annotated_plan = AnnotatedPlan { @@ -200,27 +212,51 @@ fn _annotate_plan( task_count, plan, }; - if !(root || annotated_plan.required_network_boundary.is_some()) { - return Ok(annotated_plan); - }; // The plan needs a NetworkBoundary. At this point we have all the info we need for choosing // the right size for the stage below, so what we need to do is take the calculated final // task count and propagate to all the children that will eventually be part of the stage. - fn propagate_task_count(plan: &mut AnnotatedPlan, task_count: &TaskCountAnnotation) { + fn propagate_task_count( + plan: &mut AnnotatedPlan, + task_count: &TaskCountAnnotation, + d_cfg: &DistributedConfig, + ) -> Result<(), DataFusionError> { plan.task_count = task_count.clone(); - if plan.required_network_boundary.is_none() { + if plan.required_network_boundary.is_some() { + // nothing to propagate here, all the nodes below the network boundary were already + // assigned a task count, we do not want to overwrite it. + } else if d_cfg.children_isolator_unions && plan.plan.as_any().is::() { + // Propagating through ChildrenIsolatorUnionExec is not that easy, each child will + // be executed in its own task, and therefore, they will act as if they were in executing + // in a non-distributed context. The ChildrenIsolatorUnionExec itself will make sure to + // determine which children to run and which to exclude depending on the task index in + // which it's running. + let c_i_union = ChildrenIsolatorUnionExec::from_children_and_task_counts( + plan.children.iter().map(|v| v.plan.clone()), + plan.children.iter().map(|v| v.task_count.as_usize()), + task_count.as_usize(), + )?; + for children_and_tasks in c_i_union.task_idx_map.iter() { + for (child_i, task_ctx) in children_and_tasks { + if let Some(child) = plan.children.get_mut(*child_i) { + propagate_task_count(child, &Maximum(task_ctx.task_count), d_cfg)? + }; + } + } + plan.plan = Arc::new(c_i_union); + } else { for child in &mut plan.children { - propagate_task_count(child, task_count); + propagate_task_count(child, task_count, d_cfg)?; } } + Ok(()) } if let Some(nb) = &annotated_plan.required_network_boundary { // The plan is a network boundary, so everything below it belongs to the same stage. This // means that we need to propagate the task count to all the nodes in that stage. for annotated_child in annotated_plan.children.iter_mut() { - propagate_task_count(annotated_child, &annotated_plan.task_count); + propagate_task_count(annotated_child, &annotated_plan.task_count, d_cfg)?; } // If the current plan that needs a NetworkBoundary boundary below is either a @@ -266,7 +302,7 @@ fn _annotate_plan( // If this is the root node, it means that we have just finished annotating nodes for the // subplan belonging to the head stage, so propagate the task count to all children. let task_count = annotated_plan.task_count.clone(); - propagate_task_count(&mut annotated_plan, &task_count); + propagate_task_count(&mut annotated_plan, &task_count, d_cfg)?; Ok(annotated_plan) } else { // If this is not the root node, and it's also not a network boundary, then we don't need @@ -471,16 +507,16 @@ mod tests { "#; let annotated = sql_to_annotated(query).await; assert_snapshot!(annotated, @r" - UnionExec: task_count=Desired(3) - CoalesceBatchesExec: task_count=Desired(3) - FilterExec: task_count=Desired(3) - RepartitionExec: task_count=Desired(3) - DataSourceExec: task_count=Desired(3) - ProjectionExec: task_count=Desired(3) - CoalesceBatchesExec: task_count=Desired(3) - FilterExec: task_count=Desired(3) - RepartitionExec: task_count=Desired(3) - DataSourceExec: task_count=Desired(3) + ChildrenIsolatorUnionExec: task_count=Desired(4) + CoalesceBatchesExec: task_count=Maximum(2) + FilterExec: task_count=Maximum(2) + RepartitionExec: task_count=Maximum(2) + DataSourceExec: task_count=Maximum(2) + ProjectionExec: task_count=Maximum(2) + CoalesceBatchesExec: task_count=Maximum(2) + FilterExec: task_count=Maximum(2) + RepartitionExec: task_count=Maximum(2) + DataSourceExec: task_count=Maximum(2) ") } @@ -517,6 +553,37 @@ mod tests { ") } + #[tokio::test] + async fn test_children_isolator_union() { + let query = r#" + SET distributed.children_isolator_unions = true; + SET distributed.files_per_task = 1; + SELECT "MinTemp" FROM weather WHERE "RainToday" = 'yes' + UNION ALL + SELECT "MaxTemp" FROM weather WHERE "RainToday" = 'no' + UNION ALL + SELECT "Rainfall" FROM weather WHERE "RainTomorrow" = 'yes' + "#; + let annotated = sql_to_annotated(query).await; + assert_snapshot!(annotated, @r" + ChildrenIsolatorUnionExec: task_count=Desired(4) + CoalesceBatchesExec: task_count=Maximum(1) + FilterExec: task_count=Maximum(1) + RepartitionExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + ProjectionExec: task_count=Maximum(1) + CoalesceBatchesExec: task_count=Maximum(1) + FilterExec: task_count=Maximum(1) + RepartitionExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + ProjectionExec: task_count=Maximum(2) + CoalesceBatchesExec: task_count=Maximum(2) + FilterExec: task_count=Maximum(2) + RepartitionExec: task_count=Maximum(2) + DataSourceExec: task_count=Maximum(2) + ") + } + async fn sql_to_annotated(query: &str) -> String { let config = SessionConfig::new() .with_target_partitions(4) diff --git a/src/execution_plans/children_isolator_union.rs b/src/execution_plans/children_isolator_union.rs new file mode 100644 index 00000000..6f0dad1b --- /dev/null +++ b/src/execution_plans/children_isolator_union.rs @@ -0,0 +1,541 @@ +use crate::DistributedTaskContext; +use datafusion::arrow::array::RecordBatch; +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::common::{internal_err, plan_err}; +use datafusion::error::DataFusionError; +use datafusion::execution::{RecordBatchStream, SendableRecordBatchStream, TaskContext}; +use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; +use datafusion::physical_plan::union::UnionExec; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, EmptyRecordBatchStream, ExecutionPlan, ExecutionPlanProperties, + Partitioning, PlanProperties, +}; +use futures::{Stream, StreamExt}; +use itertools::Itertools; +use std::any::Any; +use std::fmt::Formatter; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::vec; + +/// Distributed version of the vanilla [UnionExec] node that is capable of spreading the execution +/// of its children across multiple distributed tasks. +/// +/// Without [ChildrenIsolatorUnionExec], distributing a normal [UnionExec] implies scaling up +/// in partitions all the child leaf nodes and executing them all in all the assigned tasks, +/// passing a [DistributedTaskContext] so that each child knows how to distribute its work. +/// +/// With [ChildrenIsolatorUnionExec], its children are isolated per task, meaning that each +/// child will potentially be executed as if it was running in a single-node setup, and +/// [ChildrenIsolatorUnionExec] will figure out which children to execute depending on the +/// [DistributedTaskContext]. +/// +/// It's easy to think about this node in the case that the task count is equal to the number +/// of children. However, it gets a bit more complicated in case there are fewer tasks than children, +/// or more tasks than children. +/// +/// ## Case when task_count == 3 and children.len() == 3 +/// +/// ```text +/// ┌─────────────────────────────┐┌─────────────────────────────┐┌─────────────────────────────┐ +/// │ Task 1 ││ Task 2 ││ Task 3 │ +/// │┌───────────────────────────┐││┌───────────────────────────┐││┌───────────────────────────┐│ +/// ││ ChildrenIsolatorUnionExec ││││ ChildrenIsolatorUnionExec ││││ ChildrenIsolatorUnionExec ││ +/// │└───▲─────────▲─────────▲───┘││└───▲─────────▲─────────▲───┘││└───▲─────────▲─────────▲───┘│ +/// │ │ ││ │ ││ │ │ +/// │┌───┴───┐ ┌ ─│ ─ ┌ ─│ ─ ││┌ ─│ ─ ┌───┴───┐ ┌ ─│ ─ ││┌ ─│ ─ ┌ ─│ ─ ┌───┴───┐│ +/// ││Child 1│ Child 2│ Child 3│││ Child 1│ │Child 2│ Child 3│││ Child 1│ Child 2│ │Child 3││ +/// │└───────┘ └ ─ ─ └ ─ ─ ││└ ─ ─ └───────┘ └ ─ ─ ││└ ─ ─ └ ─ ─ └───────┘│ +/// └─────────────────────────────┘└─────────────────────────────┘└─────────────────────────────┘ +/// ``` +/// +/// ## Case when task_count == 2 and children.len() == 3 +/// +/// ```text +/// ┌─────────────────────────────┐┌─────────────────────────────┐ +/// │ Task 1 ││ Task 2 │ +/// │┌───────────────────────────┐││┌───────────────────────────┐│ +/// ││ ChildrenIsolatorUnionExec ││││ ChildrenIsolatorUnionExec ││ +/// │└───▲─────────▲─────────▲───┘││└───▲─────────▲─────────▲───┘│ +/// │ │ │ ││ │ │ +/// │┌───┴───┐ ┌───┴───┐ ┌ ─│ ─ ││┌ ─│ ─ ┌ ─ ┴ ─ ┐ ┌───┴───┐│ +/// ││Child 1│ │Child 2│ Child 3│││ Child 1│ Child 2 │Child 3││ +/// │└───────┘ └───────┘ └ ─ ─ ││└ ─ ─ └ ─ ─ ─ ┘ └───────┘│ +/// └─────────────────────────────┘└─────────────────────────────┘ +///``` +/// +/// ## Case when task_count == 4 and children.len() == 3 +/// +/// ```text +/// ┌─────────────────────────────┐┌─────────────────────────────┐┌─────────────────────────────┐┌─────────────────────────────┐ +/// │ Task 1 ││ Task 2 ││ Task 3 ││ Task 4 │ +/// │┌───────────────────────────┐││┌───────────────────────────┐││┌───────────────────────────┐││┌───────────────────────────┐│ +/// ││ ChildrenIsolatorUnionExec ││││ ChildrenIsolatorUnionExec ││││ ChildrenIsolatorUnionExec ││││ ChildrenIsolatorUnionExec ││ +/// │└───▲─────────▲─────────▲───┘││└───▲─────────▲─────────▲───┘││└───▲─────────▲─────────▲───┘││└───▲─────────▲─────────▲───┘│ +/// │ │ ││ │ ││ │ ││ │ │ +/// │┌───┴───┐ ┌ ─│ ─ ┌ ─│ ─ ││┌───┴───┐ ┌ ─│ ─ ┌ ─│ ─ ││┌ ─│ ─ ┌───┴───┐ ┌ ─│ ─ ││┌ ─│ ─ ┌ ─│ ─ ┌───┴───┐│ +/// ││Child 1│ Child 2│ Child 3││││Child 1│ Child 2│ Child 3│││ Child 1│ │Child 2│ Child 3│││ Child 1│ Child 2│ │Child 3││ +/// ││ (1/2) │ └ ─ ─ └ ─ ─ │││ (2/2) │ └ ─ ─ └ ─ ─ ││└ ─ ─ └───────┘ └ ─ ─ ││└ ─ ─ └ ─ ─ └───────┘│ +/// │└───────┘ ││└───────┘ ││ ││ │ +/// └─────────────────────────────┘└─────────────────────────────┘└─────────────────────────────┘└─────────────────────────────┘ +/// ``` +#[derive(Debug, Clone)] +pub struct ChildrenIsolatorUnionExec { + pub(crate) properties: PlanProperties, + pub(crate) metrics: ExecutionPlanMetricsSet, + pub(crate) children: Vec>, + pub(crate) task_idx_map: Vec< + /* outer distributed task idx */ + Vec<( + /* child index */ usize, + /* inner distributed task ctx for the isolated child*/ DistributedTaskContext, + )>, + >, +} + +impl ChildrenIsolatorUnionExec { + pub(crate) fn from_children_and_task_counts( + children: impl IntoIterator>, + children_task_count: impl IntoIterator, + task_count: usize, + ) -> Result { + let children = children.into_iter().collect_vec(); + let task_count_per_children = children_task_count.into_iter().collect_vec(); + + if children.len() != task_count_per_children.len() { + return internal_err!( + "ChildrenIsolatorUnionExec received {} children but a vec of {} positions for those children. This is a bug in the distributed planning logic, please report it", + children.len(), + task_count_per_children.len() + ); + } + + let task_idx_map = split_children(task_count_per_children, task_count)?; + + // Because different children might return a different number of partitions, and we might + // execute a different number of children in different tasks, the reality is that this node, + // depending on which task index is running, it will have a different number of partitions. + // + // We want to hide that to the outside and just advertise as many partitions as the task + // that will handle the greatest number of partitions, and just return empty streams for + // remainder partitions in tasks that will execute fewer partitions. + let mut partition_counts = vec![0; task_idx_map.len()]; + for (t, children_in_task) in task_idx_map.iter().enumerate() { + for (child_idx, _) in children_in_task { + partition_counts[t] += children[*child_idx].output_partitioning().partition_count(); + } + } + let Some(partition_count) = partition_counts.iter().max() else { + return internal_err!( + "ChildrenIsolatorUnionExec built an empty task_idx_map. This is a bug in the distributed planning logic, please report it" + ); + }; + + // It's not supper efficient to build a UnionExec just to get the properties out, but the + // other solution is to copy-paste a bunch of code from upstream for computing the properties + // of a union, so we prefer to just reuse it like this. + let mut properties = UnionExec::try_new(children.clone())?.properties().clone(); + properties.partitioning = Partitioning::UnknownPartitioning(*partition_count); + Ok(Self { + properties, + metrics: ExecutionPlanMetricsSet::default(), + children, + task_idx_map, + }) + } +} + +impl DisplayAs for ChildrenIsolatorUnionExec { + fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { + match t { + DisplayFormatType::Default | DisplayFormatType::Verbose => { + write!(f, "DistributedUnionExec:")?; + for (task_i, children_in_task) in self.task_idx_map.iter().enumerate() { + write!(f, " t{task_i}:[")?; + for (i, (child_idx, child_task_ctx)) in children_in_task.iter().enumerate() { + if child_task_ctx.task_count > 1 { + write!( + f, + "c{child_idx}({}/{})", + child_task_ctx.task_index, child_task_ctx.task_count + )?; + } else { + write!(f, "c{child_idx}")?; + } + if i < children_in_task.len() - 1 { + write!(f, ", ")?; + } + } + write!(f, "]")?; + } + + Ok(()) + } + DisplayFormatType::TreeRender => Ok(()), + } + } +} + +impl ExecutionPlan for ChildrenIsolatorUnionExec { + fn name(&self) -> &str { + "ChildrenIsolatorUnionExec" + } + + fn as_any(&self) -> &dyn Any { + self + } + + fn properties(&self) -> &PlanProperties { + &self.properties + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> datafusion::common::Result> { + if children.len() != self.children.len() { + return plan_err!( + "Number of children must match the original plan, have {} but expected {}", + children.len(), + self.children.len() + ); + } + let mut clone = self.as_ref().clone(); + clone.children = children; + Ok(Arc::new(clone)) + } + + fn children(&self) -> Vec<&Arc> { + self.children.iter().collect() + } + + fn metrics(&self) -> Option { + Some(self.metrics.clone_inner()) + } + + fn execute( + &self, + mut partition: usize, + context: Arc, + ) -> datafusion::common::Result { + let d_ctx = DistributedTaskContext::from_ctx(&context); + + let children = self.task_idx_map[d_ctx.task_index].clone(); + + let baseline_metrics = BaselineMetrics::new(&self.metrics, partition); + + let elapsed_compute = baseline_metrics.elapsed_compute().clone(); + let _timer = elapsed_compute.timer(); // record on drop + + for (child_idx, child_task_ctx) in children { + let Some(input) = self.children.get(child_idx) else { + return internal_err!("Could not find child with index {child_idx}"); + }; + // Calculate whether a partition belongs to the current partition + if partition < input.output_partitioning().partition_count() { + // We need to intercept the DistributedTaskContext and insert a modified one that + // tells the child that is running in "isolation" (see the beginning of this file + // for a longer explanation) + let context = Arc::new(TaskContext::new( + context.task_id(), + context.session_id(), + context + .session_config() + .clone() + .with_extension(Arc::new(child_task_ctx)), + context.scalar_functions().clone(), + context.aggregate_functions().clone(), + context.window_functions().clone(), + context.runtime_env(), + )); + + let stream = input.execute(partition, context)?; + + return Ok(Box::pin(ObservedStream::new( + stream, + baseline_metrics, + None, + ))); + } else { + partition -= input.output_partitioning().partition_count(); + } + } + + Ok(Box::pin(EmptyRecordBatchStream::new(self.schema()))) + } +} + +// Struct copied from https://github.com/apache/datafusion/blob/2c3566ce856bf7c87508567119bc3834f007e94b/datafusion/physical-plan/src/stream.rs#L506-L506 +// It's what allows a UnionExec to have metrics. +pub(crate) struct ObservedStream { + inner: SendableRecordBatchStream, + baseline_metrics: BaselineMetrics, + fetch: Option, + produced: usize, +} + +impl ObservedStream { + pub fn new( + inner: SendableRecordBatchStream, + baseline_metrics: BaselineMetrics, + fetch: Option, + ) -> Self { + Self { + inner, + baseline_metrics, + fetch, + produced: 0, + } + } + + fn limit_reached( + &mut self, + poll: Poll>>, + ) -> Poll>> { + let Some(fetch) = self.fetch else { return poll }; + + if self.produced >= fetch { + return Poll::Ready(None); + } + + if let Poll::Ready(Some(Ok(batch))) = &poll { + if self.produced + batch.num_rows() > fetch { + let batch = batch.slice(0, fetch.saturating_sub(self.produced)); + self.produced += batch.num_rows(); + return Poll::Ready(Some(Ok(batch))); + }; + self.produced += batch.num_rows() + } + poll + } +} + +impl RecordBatchStream for ObservedStream { + fn schema(&self) -> SchemaRef { + self.inner.schema() + } +} + +impl Stream for ObservedStream { + type Item = datafusion::common::Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let mut poll = self.inner.poll_next_unpin(cx); + if self.fetch.is_some() { + poll = self.limit_reached(poll); + } + self.baseline_metrics.record_poll(poll) + } +} + +/// Given a list of children with a different number of tasks assigned each, it redistributes them +/// and re-assign tasks numbers to them so that they fit in the provided `task_count`. +/// +/// For example, given these inputs: +/// `task_count_per_children`: [1, 1, 1] +/// `task_count`: 3 +/// It returns the following output: +/// `[[(0, 0/1)], [(1, 0/1)], [(2, 0/1)]]` +/// That means that: +/// - Task 0 will execute task 0 from child 0 +/// - Task 1 will execute task 0 from child 1 +/// - Task 2 will execute task 0 from child 2 +/// +/// Things can get more complicated if the sum of all the tasks from the children is greater than +/// the `task_count` passed: +/// +/// For example, given these inputs: +/// `task_count_per_children`: [2, 1, 1] +/// `task_count`: 3 +/// It returns the following output: +/// `[[(0, 0/1)], [(1, 0/1)], [(2, 0/1)]]` +/// As we have 3 tasks available for executing 3 children with a total sum of 2+1+1=4 tasks, we +/// need to tell that first child to be executed in 1 task. +/// +/// In this other case: +/// `task_count_per_children`: [2, 3, 1] +/// `task_count`: 5 +/// It returns the following output: +/// `[[(0, 0/2)], [(0, 1/2)], [(1, 0/2)], [(1, 1/2)], [(2, 0/1)]]` +/// Note how in this case there are 2+3+1=6 tasks to be executed, but we only have 5 tasks +/// available. We need to trim a task from somewhere, and the only candidates are child 0 and 1. +/// As child 1 is the one with the greatest task count (3), we prefer to trim the task from there, +/// as that's what will yield the most even distribution of tasks given the 5 tasks budget. +/// +/// Going to the other extreme, given this input: +/// `task_count_per_children`: [1, 1, 1, 1, 1] +/// `task_count`: 2 +/// It returns the following output: +/// `[[(0, 0/1), (1, 0/1), (2, 0/1)] , [(3, 0/1), (4, 0/1)]]` +/// In this case, we have very few `task_count` available, so we cannot afford to execute one +/// child per task. We need to group children so that one task executes several of them, and the +/// other tasks execute the several other remaining. +/// +/// The function looks pretty much like the solution of a LeetCode problem, so it's not super +/// easy to understand just be looking at the code. The best way to get a grasp of what its doing +/// is by looking at the tests. +fn split_children( + mut task_count_per_children: Vec, + task_count_budget: usize, +) -> Result< + // Task idx. This Vec will have `task_count_budget` length. + Vec< + // For this task, the child indexes and DistributedTaskContext that should be executed. + Vec<( + /* Child index */ usize, + /* Distributed task ctx for the child */ DistributedTaskContext, + )>, + >, + DataFusionError, +> { + let total_children_tasks = task_count_per_children.iter().sum::(); + if task_count_budget > total_children_tasks { + return internal_err!( + "ChildrenIsolatorUnionExec had a task count {task_count_budget}, which is greater than the sum of child task counts {total_children_tasks}. This is a bug in the distributed planning logic, please report it" + ); + } else if task_count_budget == 0 { + return internal_err!( + "ChildrenIsolatorUnionExec had a task count {task_count_budget}. This is a bug in the distributed planning logic, please report it" + ); + } + + let mut tasks_to_trim = total_children_tasks - task_count_budget; + while tasks_to_trim > 0 { + let mut max_child_task_count_idx = 0; + let mut max_child_task_count_value = 1; + for (i, child_task_count) in task_count_per_children.iter().enumerate() { + if child_task_count > &max_child_task_count_value { + max_child_task_count_idx = i; + max_child_task_count_value = *child_task_count; + } + } + if max_child_task_count_value == 1 { + break; + } + task_count_per_children[max_child_task_count_idx] -= 1; + tasks_to_trim -= 1; + } + + let total_child_tasks: usize = task_count_per_children.iter().sum(); + let base_per_task = total_child_tasks / task_count_budget; + let mut extra = total_child_tasks % task_count_budget; + + let mut result = vec![vec![]; task_count_budget]; + let mut task_idx = 0; + let mut current_task_count = 0; + let mut current_task_capacity = base_per_task; + if extra > 0 { + extra -= 1; + current_task_capacity += 1 + } + + for (child_idx, &child_task_count) in task_count_per_children.iter().enumerate() { + for task_i in 0..child_task_count { + result[task_idx].push(( + child_idx, + DistributedTaskContext { + task_index: task_i, + task_count: child_task_count, + }, + )); + current_task_count += 1; + + if current_task_count >= current_task_capacity && task_idx < task_count_budget - 1 { + task_idx += 1; + current_task_count = 0; + current_task_capacity = base_per_task; + if extra > 0 { + extra -= 1; + current_task_capacity += 1 + } + } + } + } + + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn children_split_all_1_task() -> Result<(), Box> { + assert_eq!( + split_children(vec![1, 1, 1], 3)?, + vec![ + vec![(0, ctx(0, 1))], + vec![(1, ctx(0, 1))], + vec![(2, ctx(0, 1))] + ] + ); + assert_eq!( + split_children(vec![1, 1, 1], 2)?, + vec![vec![(0, ctx(0, 1)), (1, ctx(0, 1))], vec![(2, ctx(0, 1))]] + ); + assert_eq!( + split_children(vec![1, 1, 1], 1)?, + vec![vec![(0, ctx(0, 1)), (1, ctx(0, 1)), (2, ctx(0, 1))]] + ); + Ok(()) + } + + #[test] + fn split_children_different_tasks() -> Result<(), Box> { + assert_eq!( + split_children(vec![1, 2, 3], 6)?, + vec![ + vec![(0, ctx(0, 1))], + vec![(1, ctx(0, 2))], + vec![(1, ctx(1, 2))], + vec![(2, ctx(0, 3))], + vec![(2, ctx(1, 3))], + vec![(2, ctx(2, 3))] + ] + ); + assert_eq!( + split_children(vec![1, 2, 3], 5)?, + vec![ + vec![(0, ctx(0, 1))], + vec![(1, ctx(0, 2))], + vec![(1, ctx(1, 2))], + vec![(2, ctx(0, 2))], + vec![(2, ctx(1, 2))], + ] + ); + assert_eq!( + split_children(vec![1, 2, 3], 4)?, + vec![ + vec![(0, ctx(0, 1))], + vec![(1, ctx(0, 1))], + vec![(2, ctx(0, 2))], + vec![(2, ctx(1, 2))], + ] + ); + assert_eq!( + split_children(vec![1, 2, 3], 3)?, + vec![ + vec![(0, ctx(0, 1))], + vec![(1, ctx(0, 1))], + vec![(2, ctx(0, 1))], + ] + ); + assert_eq!( + split_children(vec![1, 2, 3], 2)?, + vec![vec![(0, ctx(0, 1)), (1, ctx(0, 1))], vec![(2, ctx(0, 1))]] + ); + assert_eq!( + split_children(vec![1, 2, 3], 1)?, + vec![vec![(0, ctx(0, 1)), (1, ctx(0, 1)), (2, ctx(0, 1))]] + ); + Ok(()) + } + + fn ctx(task_index: usize, task_count: usize) -> DistributedTaskContext { + DistributedTaskContext { + task_index, + task_count, + } + } +} diff --git a/src/execution_plans/mod.rs b/src/execution_plans/mod.rs index d7719d12..9dbb027c 100644 --- a/src/execution_plans/mod.rs +++ b/src/execution_plans/mod.rs @@ -1,3 +1,4 @@ +mod children_isolator_union; mod common; mod distributed; mod metrics; @@ -5,6 +6,7 @@ mod network_coalesce; mod network_shuffle; mod partition_isolator; +pub use children_isolator_union::ChildrenIsolatorUnionExec; pub use distributed::DistributedExec; pub(crate) use metrics::MetricsWrapperExec; pub use network_coalesce::NetworkCoalesceExec; diff --git a/src/protobuf/distributed_codec.rs b/src/protobuf/distributed_codec.rs index d4c188b0..c3b8351c 100644 --- a/src/protobuf/distributed_codec.rs +++ b/src/protobuf/distributed_codec.rs @@ -1,7 +1,7 @@ use super::get_distributed_user_codecs; -use crate::NetworkBoundary; -use crate::execution_plans::NetworkCoalesceExec; +use crate::execution_plans::{ChildrenIsolatorUnionExec, NetworkCoalesceExec}; use crate::stage::{ExecutionTask, MaybeEncodedPlan, Stage}; +use crate::{DistributedTaskContext, NetworkBoundary}; use crate::{NetworkShuffleExec, PartitionIsolatorExec}; use bytes::Bytes; use datafusion::arrow::datatypes::Schema; @@ -11,6 +11,7 @@ use datafusion::error::DataFusionError; use datafusion::execution::TaskContext; use datafusion::physical_expr::EquivalenceProperties; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; +use datafusion::physical_plan::union::UnionExec; use datafusion::physical_plan::{ExecutionPlan, Partitioning, PlanProperties}; use datafusion::prelude::SessionConfig; use datafusion_proto::physical_plan::from_proto::parse_protobuf_partitioning; @@ -18,6 +19,7 @@ use datafusion_proto::physical_plan::to_proto::serialize_partitioning; use datafusion_proto::physical_plan::{ComposedPhysicalExtensionCodec, PhysicalExtensionCodec}; use datafusion_proto::protobuf; use datafusion_proto::protobuf::proto_error; +use itertools::Itertools; use prost::Message; use std::sync::Arc; use url::Url; @@ -151,6 +153,42 @@ impl PhysicalExtensionCodec for DistributedCodec { n_tasks as usize, ))) } + DistributedExecNode::ChildrenIsolatorUnion(ChildrenIsolatorUnionExecProto { + partition_count, + task_idx_map, + }) => { + // Building a UnionExec just to get the properties out of it is not the most + // efficient thing to do. However, it's the easiest way of getting the properties + // for the ChildrenIsolatorUnionExec without copy-pasting in this project + // all the machinery that builds them for UnionExec. + let mut properties = UnionExec::try_new(inputs.to_vec())?.properties().clone(); + properties.partitioning = + Partitioning::UnknownPartitioning(partition_count as usize); + + Ok(Arc::new(ChildrenIsolatorUnionExec { + properties, + metrics: Default::default(), + children: inputs.to_vec(), + task_idx_map: task_idx_map + .iter() + .map(|entry| { + entry + .child_ctx + .iter() + .map(|child_ctx| { + ( + child_ctx.child_idx as usize, + DistributedTaskContext { + task_index: child_ctx.task_idx as usize, + task_count: child_ctx.task_count as usize, + }, + ) + }) + .collect_vec() + }) + .collect(), + })) + } } } @@ -210,6 +248,30 @@ impl PhysicalExtensionCodec for DistributedCodec { node: Some(DistributedExecNode::PartitionIsolator(inner)), }; + wrapper.encode(buf).map_err(|e| proto_error(format!("{e}"))) + } else if let Some(node) = node.as_any().downcast_ref::() { + let inner = ChildrenIsolatorUnionExecProto { + partition_count: node.properties().output_partitioning().partition_count() as u64, + task_idx_map: node + .task_idx_map + .iter() + .map(|v| TaskIdxMapEntryProto { + child_ctx: v + .iter() + .map(|(child_idx, task_ctx)| ChildIdxWithTaskContextProto { + child_idx: *child_idx as u64, + task_idx: task_ctx.task_index as u64, + task_count: task_ctx.task_count as u64, + }) + .collect_vec(), + }) + .collect_vec(), + }; + + let wrapper = DistributedExecProto { + node: Some(DistributedExecNode::ChildrenIsolatorUnion(inner)), + }; + wrapper.encode(buf).map_err(|e| proto_error(format!("{e}"))) } else { Err(proto_error(format!("Unexpected plan {}", node.name()))) @@ -281,6 +343,8 @@ pub enum DistributedExecNode { NetworkCoalesceTasks(NetworkCoalesceExecProto), #[prost(message, tag = "3")] PartitionIsolator(PartitionIsolatorExecProto), + #[prost(message, tag = "4")] + ChildrenIsolatorUnion(ChildrenIsolatorUnionExecProto), } #[derive(Clone, PartialEq, ::prost::Message)] @@ -302,6 +366,30 @@ pub struct NetworkShuffleExecProto { input_stage: Option, } +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ChildrenIsolatorUnionExecProto { + #[prost(uint64, tag = "1")] + partition_count: u64, + #[prost(message, repeated, tag = "2")] + task_idx_map: Vec, +} + +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TaskIdxMapEntryProto { + #[prost(message, repeated, tag = "1")] + child_ctx: Vec, +} + +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ChildIdxWithTaskContextProto { + #[prost(uint64, tag = "1")] + child_idx: u64, + #[prost(uint64, tag = "2")] + task_idx: u64, + #[prost(uint64, tag = "3")] + task_count: u64, +} + fn new_network_hash_shuffle_exec( partitioning: Partitioning, schema: SchemaRef, @@ -587,4 +675,37 @@ mod tests { Ok(()) } + + #[test] + fn test_roundtrip_children_isolator_union() -> datafusion::common::Result<()> { + let codec = DistributedCodec; + let ctx = create_context(); + + let schema = schema_i32("h"); + let left = Arc::new(new_network_hash_shuffle_exec( + Partitioning::RoundRobinBatch(2), + schema.clone(), + dummy_stage(), + )) as Arc; + let right = Arc::new(new_network_hash_shuffle_exec( + Partitioning::RoundRobinBatch(2), + schema.clone(), + dummy_stage(), + )) as Arc; + + let plan: Arc = + Arc::new(ChildrenIsolatorUnionExec::from_children_and_task_counts( + vec![left.clone(), right.clone()], + vec![2, 2], + 4, + )?); + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let decoded = codec.try_decode(&buf, &[left, right], &ctx)?; + assert_eq!(repr(&plan), repr(&decoded)); + + Ok(()) + } } diff --git a/src/stage.rs b/src/stage.rs index b3c5bce4..14f188a7 100644 --- a/src/stage.rs +++ b/src/stage.rs @@ -133,7 +133,7 @@ impl MaybeEncodedPlan { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct DistributedTaskContext { pub task_index: usize, pub task_count: usize, diff --git a/tests/distributed_unions.rs b/tests/distributed_unions.rs new file mode 100644 index 00000000..717673c1 --- /dev/null +++ b/tests/distributed_unions.rs @@ -0,0 +1,211 @@ +#[cfg(all(feature = "integration", test))] +mod tests { + use datafusion::arrow::util::pretty::pretty_format_batches; + use datafusion::execution::TaskContext; + use datafusion::physical_plan::{ExecutionPlan, execute_stream}; + use datafusion::prelude::SessionContext; + use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::test_utils::parquet::register_parquet_tables; + use datafusion_distributed::{DefaultSessionBuilder, assert_snapshot, display_plan_ascii}; + use futures::TryStreamExt; + use std::error::Error; + use std::sync::Arc; + + #[tokio::test] + async fn more_tasks_than_children() -> Result<(), Box> { + let (ctx_distributed, _guard) = start_localhost_context(3, DefaultSessionBuilder).await; + + let query = r#" + SELECT "MinTemp", "RainToday" FROM weather WHERE "MinTemp" > 10.0 + UNION ALL + SELECT "MaxTemp", "RainToday" FROM weather WHERE "MaxTemp" < 30.0 + ORDER BY "MinTemp", "RainToday" + "#; + + let ctx = SessionContext::default(); + *ctx.state_ref().write().config_mut() = ctx_distributed.copied_config(); + register_parquet_tables(&ctx).await?; + let df = ctx.sql(query).await?; + let physical = df.create_physical_plan().await?; + + register_parquet_tables(&ctx_distributed).await?; + ctx_distributed + .sql("SET distributed.children_isolator_unions=true;") + .await?; + let df_distributed = ctx_distributed.sql(query).await?; + let physical_distributed = df_distributed.create_physical_plan().await?; + let physical_distributed_str = display_plan_ascii(physical_distributed.as_ref(), false); + + assert_snapshot!(physical_distributed_str, + @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ DistributedUnionExec: t0:[c0] t1:[c1(0/2)] t2:[c1(1/2)] + │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: MinTemp@0 > 10 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: MaxTemp@0 < 30 + │ PartitionIsolatorExec: t0:[p0,p1,__] t1:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@0 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + └────────────────────────────────────────────────── + ", + ); + + exact_same_data(ctx.task_ctx(), physical, physical_distributed).await + } + + #[tokio::test] + async fn same_children_than_tasks() -> Result<(), Box> { + let (ctx_distributed, _guard) = start_localhost_context(3, DefaultSessionBuilder).await; + + let query = r#" + SELECT "MinTemp", "RainToday" FROM weather WHERE "MinTemp" > 20.0 + UNION ALL + SELECT "MaxTemp", "RainToday" FROM weather WHERE "MaxTemp" < 25.0 + UNION ALL + SELECT "Temp9am", "RainToday" FROM weather WHERE "Temp9am" > 15.0 + ORDER BY "MinTemp", "RainToday" + "#; + + let ctx = SessionContext::default(); + *ctx.state_ref().write().config_mut() = ctx_distributed.copied_config(); + register_parquet_tables(&ctx).await?; + let df = ctx.sql(query).await?; + let physical = df.create_physical_plan().await?; + + register_parquet_tables(&ctx_distributed).await?; + ctx_distributed + .sql("SET distributed.children_isolator_unions=true;") + .await?; + let df_distributed = ctx_distributed.sql(query).await?; + let physical_distributed = df_distributed.create_physical_plan().await?; + let physical_distributed_str = display_plan_ascii(physical_distributed.as_ref(), false); + + assert_snapshot!(physical_distributed_str, + @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: MinTemp@0 > 20 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 20, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 20, required_guarantees=[] + │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: MaxTemp@0 < 25 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@0 < 25, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 25, required_guarantees=[] + │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: Temp9am@0 > 15 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@0 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] + └────────────────────────────────────────────────── + ", + ); + + exact_same_data(ctx.task_ctx(), physical, physical_distributed).await + } + + #[tokio::test] + async fn more_children_than_tasks() -> Result<(), Box> { + let (ctx_distributed, _guard) = start_localhost_context(3, DefaultSessionBuilder).await; + + let query = r#" + SELECT "MinTemp", "RainToday" FROM weather WHERE "MinTemp" > 10.0 + UNION ALL + SELECT "MaxTemp", "RainToday" FROM weather WHERE "MaxTemp" < 30.0 + UNION ALL + SELECT "Temp9am", "RainToday" FROM weather WHERE "Temp9am" > 15.0 + UNION ALL + SELECT "Temp3pm", "RainToday" FROM weather WHERE "Temp3pm" < 25.0 + UNION ALL + SELECT "Rainfall", "RainToday" FROM weather WHERE "Rainfall" > 5.0 + ORDER BY "MinTemp", "RainToday" + "#; + + let ctx = SessionContext::default(); + *ctx.state_ref().write().config_mut() = ctx_distributed.copied_config(); + register_parquet_tables(&ctx).await?; + let df = ctx.sql(query).await?; + let physical = df.create_physical_plan().await?; + + register_parquet_tables(&ctx_distributed).await?; + ctx_distributed + .sql("SET distributed.children_isolator_unions=true;") + .await?; + let df_distributed = ctx_distributed.sql(query).await?; + let physical_distributed = df_distributed.create_physical_plan().await?; + let physical_distributed_str = display_plan_ascii(physical_distributed.as_ref(), false); + + assert_snapshot!(physical_distributed_str, + @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ DistributedUnionExec: t0:[c0, c1] t1:[c2, c3] t2:[c4] + │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: MinTemp@0 > 10 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: MaxTemp@0 < 30 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@0 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: Temp9am@0 > 15 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@0 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] + │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[Temp3pm@0 as MinTemp, RainToday@1 as RainToday] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: Temp3pm@0 < 25 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[Temp3pm, RainToday], file_type=parquet, predicate=Temp3pm@0 < 25, pruning_predicate=Temp3pm_null_count@1 != row_count@2 AND Temp3pm_min@0 < 25, required_guarantees=[] + │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[Rainfall@0 as MinTemp, RainToday@1 as RainToday] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: Rainfall@0 > 5 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=Rainfall@0 > 5, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 5, required_guarantees=[] + └────────────────────────────────────────────────── + ", + ); + + exact_same_data(ctx.task_ctx(), physical, physical_distributed).await + } + + async fn exact_same_data( + task_ctx: Arc, + one: Arc, + other: Arc, + ) -> Result<(), Box> { + let batches = pretty_format_batches( + &execute_stream(one, task_ctx.clone())? + .try_collect::>() + .await?, + )?; + + let batches_distributed = pretty_format_batches( + &execute_stream(other, task_ctx)? + .try_collect::>() + .await?, + )?; + + // Verify that both plans produce the same results + assert_eq!(batches.to_string(), batches_distributed.to_string()); + Ok(()) + } +} diff --git a/tests/tpcds_plans_test.rs b/tests/tpcds_plans_test.rs index 6c201ac7..d117bda7 100644 --- a/tests/tpcds_plans_test.rs +++ b/tests/tpcds_plans_test.rs @@ -109,7 +109,7 @@ mod tests { │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq, sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END)@1 as sun_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END)@2 as mon_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END)@3 as tue_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END)@4 as wed_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END)@5 as thu_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END)@6 as fri_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)@7 as sat_sales] │ AggregateExec: mode=FinalPartitioned, gby=[d_week_seq@0 as d_week_seq], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)] - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=4 │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq2, sun_sales@1 as sun_sales2, mon_sales@2 as mon_sales2, tue_sales@3 as tue_sales2, wed_sales@4 as wed_sales2, thu_sales@5 as thu_sales2, fri_sales@6 as fri_sales2, sat_sales@7 as sat_sales2, CAST(d_week_seq@0 AS Int64) - 53 as z.d_week_seq2 - Int64(53)] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@0)], projection=[d_week_seq@1, sun_sales@2, mon_sales@3, tue_sales@4, wed_sales@5, thu_sales@6, fri_sales@7, sat_sales@8] @@ -117,7 +117,7 @@ mod tests { │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq, sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END)@1 as sun_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END)@2 as mon_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END)@3 as tue_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END)@4 as wed_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END)@5 as thu_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END)@6 as fri_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)@7 as sat_sales] │ AggregateExec: mode=FinalPartitioned, gby=[d_week_seq@0 as d_week_seq], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)] - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ CoalesceBatchesExec: target_batch_size=8192 @@ -126,7 +126,7 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] t3:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([d_week_seq@0], 3), input_partitions=3 │ AggregateExec: mode=Partial, gby=[d_week_seq@1 as d_week_seq], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)] @@ -134,20 +134,20 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, sold_date_sk@0)], projection=[d_week_seq@1, d_day_name@2, sales_price@5] │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 2 ── Tasks: t0:[p0..p11] t1:[p0..p11] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 12), input_partitions=3 │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_week_seq@1 as d_week_seq, d_day_name@2 as d_day_name, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 3 ── Tasks: t0:[p0..p11] t1:[p0..p11] t2:[p0..p11] t3:[p0..p11] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([sold_date_sk@0], 6), input_partitions=6 - │ UnionExec + │ RepartitionExec: partitioning=Hash([sold_date_sk@0], 12), input_partitions=3 + │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1(0/2)] t3:[c1(1/2)] │ ProjectionExec: expr=[ws_sold_date_sk@0 as sold_date_sk, ws_ext_sales_price@1 as sales_price] │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_ext_sales_price], file_type=parquet @@ -162,7 +162,7 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] t3:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([d_week_seq@0], 3), input_partitions=3 │ AggregateExec: mode=Partial, gby=[d_week_seq@1 as d_week_seq], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)] @@ -170,20 +170,20 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, sold_date_sk@0)], projection=[d_week_seq@1, d_day_name@2, sales_price@5] │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 6 ── Tasks: t0:[p0..p11] t1:[p0..p11] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 12), input_partitions=3 │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_week_seq@1 as d_week_seq, d_day_name@2 as d_day_name, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 7 ── Tasks: t0:[p0..p11] t1:[p0..p11] t2:[p0..p11] t3:[p0..p11] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([sold_date_sk@0], 6), input_partitions=6 - │ UnionExec + │ RepartitionExec: partitioning=Hash([sold_date_sk@0], 12), input_partitions=3 + │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1(0/2)] t3:[c1(1/2)] │ ProjectionExec: expr=[ws_sold_date_sk@0 as sold_date_sk, ws_ext_sales_price@1 as sales_price] │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_ext_sales_price], file_type=parquet @@ -492,135 +492,138 @@ mod tests { │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, id@1 ASC], preserve_partitioning=[true] │ ProjectionExec: expr=[channel@0 as channel, id@1 as id, sum(x.sales)@3 as sales, sum(x.returns_)@4 as returns_, sum(x.profit)@5 as profit] │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, id@1 as id, __grouping_id@2 as __grouping_id], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([channel@0, id@1, __grouping_id@2], 3), input_partitions=9 - │ AggregateExec: mode=Partial, gby=[(NULL as channel, NULL as id), (channel@0 as channel, NULL as id), (channel@0 as channel, id@1 as id)], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] - │ UnionExec - │ ProjectionExec: expr=[store channel as channel, concat(store, s_store_id@0) as id, sales@1 as sales, returns_@3 as returns_, profit@2 - profit_loss@4 as profit] - │ ProjectionExec: expr=[s_store_id@0 as s_store_id, sum(salesreturns.sales_price)@1 as sales, sum(salesreturns.profit)@2 as profit, sum(salesreturns.return_amt)@3 as returns_, sum(salesreturns.net_loss)@4 as profit_loss] - │ AggregateExec: mode=FinalPartitioned, gby=[s_store_id@0 as s_store_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([s_store_id@0], 3), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[s_store_id@4 as s_store_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] - │ ProjectionExec: expr=[sales_price@1 as sales_price, profit@2 as profit, return_amt@3 as return_amt, net_loss@4 as net_loss, s_store_id@0 as s_store_id] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, store_sk@0)], projection=[s_store_id@1, sales_price@4, profit@5, return_amt@6, net_loss@7] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, date_sk@1)], projection=[store_sk@2, sales_price@4, profit@5, return_amt@6, net_loss@7] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ UnionExec - │ ProjectionExec: expr=[ss_store_sk@1 as store_sk, ss_sold_date_sk@0 as date_sk, ss_ext_sales_price@2 as sales_price, CAST(ss_net_profit@3 AS Decimal128(7, 2)) as profit, Some(0),7,2 as return_amt, Some(0),7,2 as net_loss] - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet - │ ProjectionExec: expr=[sr_store_sk@1 as store_sk, sr_returned_date_sk@0 as date_sk, Some(0),7,2 as sales_price, Some(0),7,2 as profit, sr_return_amt@2 as return_amt, CAST(sr_net_loss@3 AS Decimal128(7, 2)) as net_loss] - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_store_sk, sr_return_amt, sr_net_loss], file_type=parquet - │ ProjectionExec: expr=[catalog channel as channel, concat(catalog_page, cp_catalog_page_id@0) as id, sales@1 as sales, returns_@3 as returns_, profit@2 - profit_loss@4 as profit] - │ ProjectionExec: expr=[cp_catalog_page_id@0 as cp_catalog_page_id, sum(salesreturns.sales_price)@1 as sales, sum(salesreturns.profit)@2 as profit, sum(salesreturns.return_amt)@3 as returns_, sum(salesreturns.net_loss)@4 as profit_loss] - │ AggregateExec: mode=FinalPartitioned, gby=[cp_catalog_page_id@0 as cp_catalog_page_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ ProjectionExec: expr=[web channel as channel, concat(web_site, web_site_id@0) as id, sales@1 as sales, returns_@3 as returns_, profit@2 - profit_loss@4 as profit] - │ ProjectionExec: expr=[web_site_id@0 as web_site_id, sum(salesreturns.sales_price)@1 as sales, sum(salesreturns.profit)@2 as profit, sum(salesreturns.return_amt)@3 as returns_, sum(salesreturns.net_loss)@4 as profit_loss] - │ AggregateExec: mode=FinalPartitioned, gby=[web_site_id@0 as web_site_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([web_site_id@0], 3), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[web_site_id@4 as web_site_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] - │ ProjectionExec: expr=[sales_price@1 as sales_price, profit@2 as profit, return_amt@3 as return_amt, net_loss@4 as net_loss, web_site_id@0 as web_site_id] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_site.web_site_sk AS Float64)@2, wsr_web_site_sk@0)], projection=[web_site_id@1, sales_price@4, profit@5, return_amt@6, net_loss@7] - │ CoalescePartitionsExec - │ [Stage 7] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, date_sk@1)], projection=[wsr_web_site_sk@2, sales_price@4, profit@5, return_amt@6, net_loss@7] - │ CoalescePartitionsExec - │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ UnionExec - │ ProjectionExec: expr=[ws_web_site_sk@1 as wsr_web_site_sk, ws_sold_date_sk@0 as date_sk, ws_ext_sales_price@2 as sales_price, ws_net_profit@3 as profit, Some(0),7,2 as return_amt, Some(0),7,2 as net_loss] - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_web_site_sk, ws_ext_sales_price, ws_net_profit], file_type=parquet - │ ProjectionExec: expr=[ws_web_site_sk@6 as wsr_web_site_sk, wr_returned_date_sk@0 as date_sk, Some(0),7,2 as sales_price, Some(0),7,2 as profit, wr_return_amt@3 as return_amt, wr_net_loss@4 as net_loss] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(wr_item_sk@1, ws_item_sk@0), (wr_order_number@2, ws_order_number@2)] - │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] - │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_id@1 as s_store_id, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cp_catalog_page_id@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[cp_catalog_page_id@4 as cp_catalog_page_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(page_sk@0, CAST(catalog_page.cp_catalog_page_sk AS Float64)@2)], projection=[sales_price@1, profit@2, return_amt@3, net_loss@4, cp_catalog_page_id@6] - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=1 - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p5] + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([channel@0, id@1, __grouping_id@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[(NULL as channel, NULL as id), (channel@0 as channel, NULL as id), (channel@0 as channel, id@1 as id)], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ ProjectionExec: expr=[store channel as channel, concat(store, s_store_id@0) as id, sales@1 as sales, returns_@3 as returns_, profit@2 - profit_loss@4 as profit] + │ ProjectionExec: expr=[s_store_id@0 as s_store_id, sum(salesreturns.sales_price)@1 as sales, sum(salesreturns.profit)@2 as profit, sum(salesreturns.return_amt)@3 as returns_, sum(salesreturns.net_loss)@4 as profit_loss] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_id@0 as s_store_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([s_store_id@0], 3), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[s_store_id@4 as s_store_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] + │ ProjectionExec: expr=[sales_price@1 as sales_price, profit@2 as profit, return_amt@3 as return_amt, net_loss@4 as net_loss, s_store_id@0 as s_store_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, store_sk@0)], projection=[s_store_id@1, sales_price@4, profit@5, return_amt@6, net_loss@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, date_sk@1)], projection=[store_sk@2, sales_price@4, profit@5, return_amt@6, net_loss@7] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DistributedUnionExec: t0:[c0, c1] + │ ProjectionExec: expr=[ss_store_sk@1 as store_sk, ss_sold_date_sk@0 as date_sk, ss_ext_sales_price@2 as sales_price, CAST(ss_net_profit@3 AS Decimal128(7, 2)) as profit, Some(0),7,2 as return_amt, Some(0),7,2 as net_loss] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet + │ ProjectionExec: expr=[sr_store_sk@1 as store_sk, sr_returned_date_sk@0 as date_sk, Some(0),7,2 as sales_price, Some(0),7,2 as profit, sr_return_amt@2 as return_amt, CAST(sr_net_loss@3 AS Decimal128(7, 2)) as net_loss] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_store_sk, sr_return_amt, sr_net_loss], file_type=parquet + │ ProjectionExec: expr=[catalog channel as channel, concat(catalog_page, cp_catalog_page_id@0) as id, sales@1 as sales, returns_@3 as returns_, profit@2 - profit_loss@4 as profit] + │ ProjectionExec: expr=[cp_catalog_page_id@0 as cp_catalog_page_id, sum(salesreturns.sales_price)@1 as sales, sum(salesreturns.profit)@2 as profit, sum(salesreturns.return_amt)@3 as returns_, sum(salesreturns.net_loss)@4 as profit_loss] + │ AggregateExec: mode=FinalPartitioned, gby=[cp_catalog_page_id@0 as cp_catalog_page_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[web channel as channel, concat(web_site, web_site_id@0) as id, sales@1 as sales, returns_@3 as returns_, profit@2 - profit_loss@4 as profit] + │ ProjectionExec: expr=[web_site_id@0 as web_site_id, sum(salesreturns.sales_price)@1 as sales, sum(salesreturns.profit)@2 as profit, sum(salesreturns.return_amt)@3 as returns_, sum(salesreturns.net_loss)@4 as profit_loss] + │ AggregateExec: mode=FinalPartitioned, gby=[web_site_id@0 as web_site_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([web_site_id@0], 3), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[web_site_id@4 as web_site_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] + │ ProjectionExec: expr=[sales_price@1 as sales_price, profit@2 as profit, return_amt@3 as return_amt, net_loss@4 as net_loss, web_site_id@0 as web_site_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_site.web_site_sk AS Float64)@2, wsr_web_site_sk@0)], projection=[web_site_id@1, sales_price@4, profit@5, return_amt@6, net_loss@7] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, date_sk@1)], projection=[wsr_web_site_sk@2, sales_price@4, profit@5, return_amt@6, net_loss@7] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DistributedUnionExec: t0:[c0, c1] + │ ProjectionExec: expr=[ws_web_site_sk@1 as wsr_web_site_sk, ws_sold_date_sk@0 as date_sk, ws_ext_sales_price@2 as sales_price, ws_net_profit@3 as profit, Some(0),7,2 as return_amt, Some(0),7,2 as net_loss] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_web_site_sk, ws_ext_sales_price, ws_net_profit], file_type=parquet + │ ProjectionExec: expr=[ws_web_site_sk@6 as wsr_web_site_sk, wr_returned_date_sk@0 as date_sk, Some(0),7,2 as sales_price, Some(0),7,2 as profit, wr_return_amt@3 as return_amt, wr_net_loss@4 as net_loss] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(wr_item_sk@1, ws_item_sk@0), (wr_order_number@2, ws_order_number@2)] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_id@1 as s_store_id, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([page_sk@0], 6), input_partitions=6 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, date_sk@1)], projection=[page_sk@2, sales_price@4, profit@5, return_amt@6, net_loss@7] - │ CoalescePartitionsExec - │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ UnionExec - │ ProjectionExec: expr=[cs_catalog_page_sk@1 as page_sk, cs_sold_date_sk@0 as date_sk, cs_ext_sales_price@2 as sales_price, cs_net_profit@3 as profit, Some(0),7,2 as return_amt, Some(0),7,2 as net_loss] - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_catalog_page_sk, cs_ext_sales_price, cs_net_profit], file_type=parquet - │ ProjectionExec: expr=[cr_catalog_page_sk@1 as page_sk, CAST(cr_returned_date_sk@0 AS Float64) as date_sk, Some(0),7,2 as sales_price, Some(0),7,2 as profit, cr_return_amount@2 as return_amt, cr_net_loss@3 as net_loss] - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_catalog_page_sk, cr_return_amount, cr_net_loss], file_type=parquet + │ RepartitionExec: partitioning=Hash([cp_catalog_page_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cp_catalog_page_id@4 as cp_catalog_page_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(page_sk@0, CAST(catalog_page.cp_catalog_page_sk AS Float64)@2)], projection=[sales_price@1, profit@2, return_amt@3, net_loss@4, cp_catalog_page_id@6] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=1 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, projection=[d_date_sk@0] + ┌───── Stage 4 ── Tasks: t0:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([page_sk@0], 6), input_partitions=6 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, date_sk@1)], projection=[page_sk@2, sales_price@4, profit@5, return_amt@6, net_loss@7] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DistributedUnionExec: t0:[c0, c1] + │ ProjectionExec: expr=[cs_catalog_page_sk@1 as page_sk, cs_sold_date_sk@0 as date_sk, cs_ext_sales_price@2 as sales_price, cs_net_profit@3 as profit, Some(0),7,2 as return_amt, Some(0),7,2 as net_loss] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_catalog_page_sk, cs_ext_sales_price, cs_net_profit], file_type=parquet + │ ProjectionExec: expr=[cr_catalog_page_sk@1 as page_sk, CAST(cr_returned_date_sk@0 AS Float64) as date_sk, Some(0),7,2 as sales_price, Some(0),7,2 as profit, cr_return_amount@2 as return_amt, cr_net_loss@3 as net_loss] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_catalog_page_sk, cr_return_amount, cr_net_loss], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(catalog_page.cp_catalog_page_sk AS Float64)@2], 6), input_partitions=3 + │ ProjectionExec: expr=[cp_catalog_page_sk@0 as cp_catalog_page_sk, cp_catalog_page_id@1 as cp_catalog_page_id, CAST(cp_catalog_page_sk@0 AS Float64) as CAST(catalog_page.cp_catalog_page_sk AS Float64)] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-3.parquet]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(catalog_page.cp_catalog_page_sk AS Float64)@2], 6), input_partitions=3 - │ ProjectionExec: expr=[cp_catalog_page_sk@0 as cp_catalog_page_sk, cp_catalog_page_id@1 as cp_catalog_page_id, CAST(cp_catalog_page_sk@0 AS Float64) as CAST(catalog_page.cp_catalog_page_sk AS Float64)] + ┌───── Stage 7 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[web_site_sk@0 as web_site_sk, web_site_id@1 as web_site_id, CAST(web_site_sk@0 AS Float64) as CAST(web_site.web_site_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-3.parquet]]}, projection=[web_site_sk, web_site_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, projection=[d_date_sk@0] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-3.parquet]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([wr_item_sk@1, wr_order_number@2], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_item_sk@0, ws_order_number@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_item_sk, ws_web_site_sk, ws_order_number], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p1] t1:[p2..p3] - │ ProjectionExec: expr=[web_site_sk@0 as web_site_sk, web_site_id@1 as web_site_id, CAST(web_site_sk@0 AS Float64) as CAST(web_site.web_site_sk AS Float64)] - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-3.parquet]]}, projection=[web_site_sk, web_site_id], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([wr_item_sk@1, wr_order_number@2], 3), input_partitions=3 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_item_sk@0, ws_order_number@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_item_sk, ws_web_site_sk, ws_order_number], file_type=parquet - └────────────────────────────────────────────────── "); Ok(()) } @@ -1428,26 +1431,7 @@ mod tests { │ ProjectionExec: expr=[avg(sq2.quantity * sq2.list_price)@0 as average_sales] │ AggregateExec: mode=Final, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] │ CoalescePartitionsExec - │ AggregateExec: mode=Partial, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] - │ UnionExec - │ ProjectionExec: expr=[ss_quantity@0 as quantity, ss_list_price@1 as list_price] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_quantity@3, ss_list_price@4] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ ProjectionExec: expr=[cs_quantity@0 as quantity, cs_list_price@1 as list_price] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_quantity@3, cs_list_price@4] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ ProjectionExec: expr=[ws_quantity@0 as quantity, ws_list_price@1 as list_price] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_quantity@3, ws_list_price@4] - │ CoalescePartitionsExec - │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ [Stage 4] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 │ AggregateExec: mode=FinalPartitioned, gby=[i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id], aggr=[sum(store_sales.ss_quantity * store_sales.ss_list_price), count(Int64(1))] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 3), input_partitions=3 @@ -1458,12 +1442,12 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_quantity@4, ss_list_price@5, i_brand_id@6, i_class_id@7, i_category_id@8] │ CoalescePartitionsExec - │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ ProjectionExec: expr=[ss_sold_date_sk@3 as ss_sold_date_sk, ss_item_sk@4 as ss_item_sk, ss_quantity@5 as ss_quantity, ss_list_price@6 as ss_list_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ss_sold_date_sk@4, ss_item_sk@5, ss_quantity@6, ss_list_price@7] - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ ProjectionExec: expr=[i_item_sk@0 as ss_item_sk] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(brand_id@0, i_brand_id@1), (class_id@1, i_class_id@2), (category_id@2, i_category_id@3)], projection=[i_item_sk@3] @@ -1474,12 +1458,12 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d3.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] │ CoalescePartitionsExec - │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ ProjectionExec: expr=[ws_sold_date_sk@3 as ws_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ws_sold_date_sk@4] - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ AggregateExec: mode=SinglePartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true @@ -1487,12 +1471,12 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d2.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] │ CoalescePartitionsExec - │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ ProjectionExec: expr=[cs_sold_date_sk@3 as cs_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, cs_sold_date_sk@4] - │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ AggregateExec: mode=FinalPartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 3), input_partitions=3 @@ -1501,12 +1485,12 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] │ CoalescePartitionsExec - │ [Stage 13] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 14] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ ProjectionExec: expr=[ss_sold_date_sk@3 as ss_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ss_sold_date_sk@4] - │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] │ ProjectionExec: expr=[catalog as channel, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)@3 as sales, count(Int64(1))@4 as number_sales] @@ -1515,26 +1499,7 @@ mod tests { │ ProjectionExec: expr=[avg(sq2.quantity * sq2.list_price)@0 as average_sales] │ AggregateExec: mode=Final, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] │ CoalescePartitionsExec - │ AggregateExec: mode=Partial, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] - │ UnionExec - │ ProjectionExec: expr=[ss_quantity@0 as quantity, ss_list_price@1 as list_price] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_quantity@3, ss_list_price@4] - │ CoalescePartitionsExec - │ [Stage 16] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ ProjectionExec: expr=[cs_quantity@0 as quantity, cs_list_price@1 as list_price] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_quantity@3, cs_list_price@4] - │ CoalescePartitionsExec - │ [Stage 17] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ ProjectionExec: expr=[ws_quantity@0 as quantity, ws_list_price@1 as list_price] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_quantity@3, ws_list_price@4] - │ CoalescePartitionsExec - │ [Stage 18] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ [Stage 20] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 │ AggregateExec: mode=FinalPartitioned, gby=[i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id], aggr=[sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price), count(Int64(1))] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 3), input_partitions=3 @@ -1545,12 +1510,12 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_item_sk@3, cs_quantity@4, cs_list_price@5, i_brand_id@6, i_class_id@7, i_category_id@8] │ CoalescePartitionsExec - │ [Stage 19] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 21] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ ProjectionExec: expr=[cs_sold_date_sk@3 as cs_sold_date_sk, cs_item_sk@4 as cs_item_sk, cs_quantity@5 as cs_quantity, cs_list_price@6 as cs_list_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, cs_sold_date_sk@4, cs_item_sk@5, cs_quantity@6, cs_list_price@7] - │ [Stage 20] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 21] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 22] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 23] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ ProjectionExec: expr=[i_item_sk@0 as ss_item_sk] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(brand_id@0, i_brand_id@1), (class_id@1, i_class_id@2), (category_id@2, i_category_id@3)], projection=[i_item_sk@3] @@ -1561,12 +1526,12 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d3.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] │ CoalescePartitionsExec - │ [Stage 22] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 24] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ ProjectionExec: expr=[ws_sold_date_sk@3 as ws_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ws_sold_date_sk@4] - │ [Stage 23] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 24] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 25] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 26] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ AggregateExec: mode=SinglePartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true @@ -1574,12 +1539,12 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d2.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] │ CoalescePartitionsExec - │ [Stage 25] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 27] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ ProjectionExec: expr=[cs_sold_date_sk@3 as cs_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, cs_sold_date_sk@4] - │ [Stage 26] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 27] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 28] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 29] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ AggregateExec: mode=FinalPartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 3), input_partitions=3 @@ -1588,12 +1553,12 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] │ CoalescePartitionsExec - │ [Stage 28] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 30] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ ProjectionExec: expr=[ss_sold_date_sk@3 as ss_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ss_sold_date_sk@4] - │ [Stage 29] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 30] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 31] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 32] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] │ ProjectionExec: expr=[web as channel, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, sum(web_sales.ws_quantity * web_sales.ws_list_price)@3 as sales, count(Int64(1))@4 as number_sales] @@ -1602,26 +1567,7 @@ mod tests { │ ProjectionExec: expr=[avg(sq2.quantity * sq2.list_price)@0 as average_sales] │ AggregateExec: mode=Final, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] │ CoalescePartitionsExec - │ AggregateExec: mode=Partial, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] - │ UnionExec - │ ProjectionExec: expr=[ss_quantity@0 as quantity, ss_list_price@1 as list_price] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_quantity@3, ss_list_price@4] - │ CoalescePartitionsExec - │ [Stage 31] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ ProjectionExec: expr=[cs_quantity@0 as quantity, cs_list_price@1 as list_price] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_quantity@3, cs_list_price@4] - │ CoalescePartitionsExec - │ [Stage 32] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ ProjectionExec: expr=[ws_quantity@0 as quantity, ws_list_price@1 as list_price] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_quantity@3, ws_list_price@4] - │ CoalescePartitionsExec - │ [Stage 33] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ [Stage 36] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 │ AggregateExec: mode=FinalPartitioned, gby=[i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id], aggr=[sum(web_sales.ws_quantity * web_sales.ws_list_price), count(Int64(1))] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 3), input_partitions=3 @@ -1632,12 +1578,12 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_quantity@4, ws_list_price@5, i_brand_id@6, i_class_id@7, i_category_id@8] │ CoalescePartitionsExec - │ [Stage 34] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 37] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ ProjectionExec: expr=[ws_sold_date_sk@3 as ws_sold_date_sk, ws_item_sk@4 as ws_item_sk, ws_quantity@5 as ws_quantity, ws_list_price@6 as ws_list_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ws_sold_date_sk@4, ws_item_sk@5, ws_quantity@6, ws_list_price@7] - │ [Stage 35] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 36] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 38] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 39] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ ProjectionExec: expr=[i_item_sk@0 as ss_item_sk] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(brand_id@0, i_brand_id@1), (class_id@1, i_class_id@2), (category_id@2, i_category_id@3)], projection=[i_item_sk@3] @@ -1648,12 +1594,12 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d3.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] │ CoalescePartitionsExec - │ [Stage 37] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 40] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ ProjectionExec: expr=[ws_sold_date_sk@3 as ws_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ws_sold_date_sk@4] - │ [Stage 38] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 39] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 41] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 42] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ AggregateExec: mode=SinglePartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true @@ -1661,12 +1607,12 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d2.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] │ CoalescePartitionsExec - │ [Stage 40] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 43] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ ProjectionExec: expr=[cs_sold_date_sk@3 as cs_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, cs_sold_date_sk@4] - │ [Stage 41] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 42] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 44] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 45] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ AggregateExec: mode=FinalPartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 3), input_partitions=3 @@ -1675,40 +1621,62 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] │ CoalescePartitionsExec - │ [Stage 43] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 46] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ ProjectionExec: expr=[ss_sold_date_sk@3 as ss_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ss_sold_date_sk@4] - │ [Stage 44] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 45] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 47] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 48] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ ProjectionExec: expr=[ss_quantity@0 as quantity, ss_list_price@1 as list_price] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_quantity@3, ss_list_price@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[cs_quantity@0 as quantity, cs_list_price@1 as list_price] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_quantity@3, cs_list_price@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ws_quantity@0 as quantity, ws_list_price@1 as list_price] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_quantity@3, ws_list_price@4] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 11, projection=[d_date_sk@0] @@ -1716,20 +1684,20 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=3 │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] @@ -1737,20 +1705,20 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 3), input_partitions=3 │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] @@ -1758,20 +1726,20 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 13 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 3), input_partitions=3 │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 13 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] @@ -1779,44 +1747,66 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 15 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 15 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=3 │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 17 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 18 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 19 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 20 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ ProjectionExec: expr=[ss_quantity@0 as quantity, ss_list_price@1 as list_price] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_quantity@3, ss_list_price@4] + │ CoalescePartitionsExec + │ [Stage 17] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[cs_quantity@0 as quantity, cs_list_price@1 as list_price] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_quantity@3, cs_list_price@4] + │ CoalescePartitionsExec + │ [Stage 18] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ws_quantity@0 as quantity, ws_list_price@1 as list_price] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_quantity@3, ws_list_price@4] + │ CoalescePartitionsExec + │ [Stage 19] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 17 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 18 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 19 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 21 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 11, projection=[d_date_sk@0] @@ -1824,20 +1814,20 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 20 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 22 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 21 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 23 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 3), input_partitions=3 │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 22 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 24 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] @@ -1845,20 +1835,20 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 23 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 25 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 24 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 26 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 3), input_partitions=3 │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 25 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 27 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] @@ -1866,20 +1856,20 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 26 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 28 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 27 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 29 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 3), input_partitions=3 │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 28 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 30 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] @@ -1887,44 +1877,66 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 29 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 31 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 30 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 32 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=3 │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 31 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 32 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 33 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 34 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 36 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ ProjectionExec: expr=[ss_quantity@0 as quantity, ss_list_price@1 as list_price] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_quantity@3, ss_list_price@4] + │ CoalescePartitionsExec + │ [Stage 33] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[cs_quantity@0 as quantity, cs_list_price@1 as list_price] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_quantity@3, cs_list_price@4] + │ CoalescePartitionsExec + │ [Stage 34] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ws_quantity@0 as quantity, ws_list_price@1 as list_price] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_quantity@3, ws_list_price@4] + │ CoalescePartitionsExec + │ [Stage 35] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 33 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 34 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 35 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 37 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 11, projection=[d_date_sk@0] @@ -1932,20 +1944,20 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 35 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 38 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 36 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 39 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 3), input_partitions=3 │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 37 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 40 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] @@ -1953,20 +1965,20 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 38 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 41 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 39 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 42 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 3), input_partitions=3 │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 40 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 43 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] @@ -1974,20 +1986,20 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 41 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 44 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 42 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 45 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 3), input_partitions=3 │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 43 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 46 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] @@ -1995,14 +2007,14 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 >= 1999 AND d_year@1 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 44 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 47 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([i_item_sk@0], 3), input_partitions=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 45 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 48 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=3 │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] @@ -3117,171 +3129,174 @@ mod tests { assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [i_item_id@0 ASC, s_state@1 ASC], fetch=100 - │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC], preserve_partitioning=[true] - │ UnionExec - │ ProjectionExec: expr=[i_item_id@0 as i_item_id, CAST(s_state@1 AS Utf8) as s_state, 0 as g_state, avg(results.agg1)@2 as agg1, avg(results.agg2)@3 as agg2, avg(results.agg3)@4 as agg3, avg(results.agg4)@5 as agg4] - │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, s_state@1 as s_state], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)], ordering_mode=PartiallySorted([1]) - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([i_item_id@0, s_state@1], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[i_item_id@0 as i_item_id, s_state@1 as s_state], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)], ordering_mode=PartiallySorted([1]) - │ ProjectionExec: expr=[i_item_id@5 as i_item_id, s_state@4 as s_state, ss_quantity@0 as agg1, ss_list_price@1 as agg2, ss_coupon_amt@3 as agg3, ss_sales_price@2 as agg4] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_quantity@1, ss_list_price@2, ss_sales_price@3, ss_coupon_amt@4, s_state@5, i_item_id@7] - │ CoalescePartitionsExec - │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_quantity@2 as ss_quantity, ss_list_price@3 as ss_list_price, ss_sales_price@4 as ss_sales_price, ss_coupon_amt@5 as ss_coupon_amt, s_state@0 as s_state] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, ss_store_sk@1)], projection=[s_state@1, ss_item_sk@3, ss_quantity@5, ss_list_price@6, ss_sales_price@7, ss_coupon_amt@8] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_quantity@5, ss_list_price@6, ss_sales_price@7, ss_coupon_amt@8] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, ss_cdemo_sk@2)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_store_sk@5, ss_quantity@6, ss_list_price@7, ss_sales_price@8, ss_coupon_amt@9] - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ ProjectionExec: expr=[i_item_id@0 as i_item_id, NULL as s_state, 1 as g_state, avg(results.agg1)@1 as agg1, avg(results.agg2)@2 as agg2, avg(results.agg3)@3 as agg3, avg(results.agg4)@4 as agg4] - │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[i_item_id@0 as i_item_id], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)] - │ ProjectionExec: expr=[i_item_id@4 as i_item_id, ss_quantity@0 as agg1, ss_list_price@1 as agg2, ss_coupon_amt@3 as agg3, ss_sales_price@2 as agg4] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_quantity@1, ss_list_price@2, ss_sales_price@3, ss_coupon_amt@4, i_item_id@6] - │ CoalescePartitionsExec - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@1)], projection=[ss_item_sk@2, ss_quantity@4, ss_list_price@5, ss_sales_price@6, ss_coupon_amt@7] - │ CoalescePartitionsExec - │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_quantity@5, ss_list_price@6, ss_sales_price@7, ss_coupon_amt@8] - │ CoalescePartitionsExec - │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, ss_cdemo_sk@2)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_store_sk@5, ss_quantity@6, ss_list_price@7, ss_sales_price@8, ss_coupon_amt@9] - │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ ProjectionExec: expr=[NULL as i_item_id, NULL as s_state, 1 as g_state, avg(results.agg1)@0 as agg1, avg(results.agg2)@1 as agg2, avg(results.agg3)@2 as agg3, avg(results.agg4)@3 as agg4] - │ AggregateExec: mode=Final, gby=[], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)] - │ CoalescePartitionsExec - │ AggregateExec: mode=Partial, gby=[], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)] - │ ProjectionExec: expr=[ss_quantity@0 as agg1, ss_list_price@1 as agg2, ss_coupon_amt@3 as agg3, ss_sales_price@2 as agg4] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_quantity@1, ss_list_price@2, ss_sales_price@3, ss_coupon_amt@4] - │ CoalescePartitionsExec - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@1)], projection=[ss_item_sk@2, ss_quantity@4, ss_list_price@5, ss_sales_price@6, ss_coupon_amt@7] - │ CoalescePartitionsExec - │ [Stage 9] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_quantity@5, ss_list_price@6, ss_sales_price@7, ss_coupon_amt@8] - │ CoalescePartitionsExec - │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, ss_cdemo_sk@2)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_store_sk@5, ss_quantity@6, ss_list_price@7, ss_sales_price@8, ss_coupon_amt@9] - │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ [Stage 13] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_state@1 as s_state, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: s_state@1 = TN - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@1 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 = 2002, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=3 - │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@1 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] - └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 = 2002, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=3 - │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] - └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@1 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] - └────────────────────────────────────────────────── - ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 = 2002, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - └────────────────────────────────────────────────── - ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=3 - │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] - └────────────────────────────────────────────────── - ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] - └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC], preserve_partitioning=[true] + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, CAST(s_state@1 AS Utf8) as s_state, 0 as g_state, avg(results.agg1)@2 as agg1, avg(results.agg2)@3 as agg2, avg(results.agg3)@4 as agg3, avg(results.agg4)@5 as agg4] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, s_state@1 as s_state], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)], ordering_mode=PartiallySorted([1]) + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0, s_state@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@0 as i_item_id, s_state@1 as s_state], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)], ordering_mode=PartiallySorted([1]) + │ ProjectionExec: expr=[i_item_id@5 as i_item_id, s_state@4 as s_state, ss_quantity@0 as agg1, ss_list_price@1 as agg2, ss_coupon_amt@3 as agg3, ss_sales_price@2 as agg4] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_quantity@1, ss_list_price@2, ss_sales_price@3, ss_coupon_amt@4, s_state@5, i_item_id@7] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_quantity@2 as ss_quantity, ss_list_price@3 as ss_list_price, ss_sales_price@4 as ss_sales_price, ss_coupon_amt@5 as ss_coupon_amt, s_state@0 as s_state] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, ss_store_sk@1)], projection=[s_state@1, ss_item_sk@3, ss_quantity@5, ss_list_price@6, ss_sales_price@7, ss_coupon_amt@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_quantity@5, ss_list_price@6, ss_sales_price@7, ss_coupon_amt@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, ss_cdemo_sk@2)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_store_sk@5, ss_quantity@6, ss_list_price@7, ss_sales_price@8, ss_coupon_amt@9] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, NULL as s_state, 1 as g_state, avg(results.agg1)@1 as agg1, avg(results.agg2)@2 as agg2, avg(results.agg3)@3 as agg3, avg(results.agg4)@4 as agg4] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@0 as i_item_id], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)] + │ ProjectionExec: expr=[i_item_id@4 as i_item_id, ss_quantity@0 as agg1, ss_list_price@1 as agg2, ss_coupon_amt@3 as agg3, ss_sales_price@2 as agg4] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_quantity@1, ss_list_price@2, ss_sales_price@3, ss_coupon_amt@4, i_item_id@6] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@1)], projection=[ss_item_sk@2, ss_quantity@4, ss_list_price@5, ss_sales_price@6, ss_coupon_amt@7] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_quantity@5, ss_list_price@6, ss_sales_price@7, ss_coupon_amt@8] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, ss_cdemo_sk@2)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_store_sk@5, ss_quantity@6, ss_list_price@7, ss_sales_price@8, ss_coupon_amt@9] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[NULL as i_item_id, NULL as s_state, 1 as g_state, avg(results.agg1)@0 as agg1, avg(results.agg2)@1 as agg2, avg(results.agg3)@2 as agg3, avg(results.agg4)@3 as agg4] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)] + │ ProjectionExec: expr=[ss_quantity@0 as agg1, ss_list_price@1 as agg2, ss_coupon_amt@3 as agg3, ss_sales_price@2 as agg4] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_quantity@1, ss_list_price@2, ss_sales_price@3, ss_coupon_amt@4] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@1)], projection=[ss_item_sk@2, ss_quantity@4, ss_list_price@5, ss_sales_price@6, ss_coupon_amt@7] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_quantity@5, ss_list_price@6, ss_sales_price@7, ss_coupon_amt@8] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, ss_cdemo_sk@2)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_store_sk@5, ss_quantity@6, ss_list_price@7, ss_sales_price@8, ss_coupon_amt@9] + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_state@1 as s_state, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_state@1 = TN + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@1 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=3 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@1 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=3 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@1 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=3 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── "); Ok(()) } @@ -4006,166 +4021,172 @@ mod tests { │ RepartitionExec: partitioning=Hash([lochierarchy@4, CASE WHEN t_class@3 = 0 THEN i_category@1 END], 3), input_partitions=3 │ ProjectionExec: expr=[gross_margin@0 as gross_margin, i_category@1 as i_category, i_class@2 as i_class, t_class@4 as t_class, lochierarchy@5 as lochierarchy] │ AggregateExec: mode=FinalPartitioned, gby=[gross_margin@0 as gross_margin, i_category@1 as i_category, i_class@2 as i_class, t_category@3 as t_category, t_class@4 as t_class, lochierarchy@5 as lochierarchy], aggr=[] + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([gross_margin@0, i_category@1, i_class@2, t_category@3, t_class@4, lochierarchy@5], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[gross_margin@0 as gross_margin, i_category@1 as i_category, i_class@2 as i_class, t_category@3 as t_category, t_class@4 as t_class, lochierarchy@5 as lochierarchy], aggr=[], ordering_mode=PartiallySorted([3]) + │ DistributedUnionExec: t0:[c0] t1:[c1] + │ ProjectionExec: expr=[gross_margin@0 as gross_margin, CAST(i_category@1 AS Utf8) as i_category, i_class@2 as i_class, t_category@3 as t_category, t_class@4 as t_class, lochierarchy@5 as lochierarchy] + │ AggregateExec: mode=FinalPartitioned, gby=[gross_margin@0 as gross_margin, i_category@1 as i_category, i_class@2 as i_class, t_category@3 as t_category, t_class@4 as t_class, lochierarchy@5 as lochierarchy], aggr=[], ordering_mode=PartiallySorted([3]) + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[CAST(sum(results.ss_net_profit)@0 AS Float64) / CAST(sum(results.ss_ext_sales_price)@1 AS Float64) as gross_margin, NULL as i_category, NULL as i_class, 1 as t_category, 1 as t_class, 2 as lochierarchy] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(results.ss_net_profit), sum(results.ss_ext_sales_price)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(results.ss_net_profit), sum(results.ss_ext_sales_price)] + │ ProjectionExec: expr=[sum(store_sales.ss_net_profit)@2 as ss_net_profit, sum(store_sales.ss_ext_sales_price)@3 as ss_ext_sales_price] + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_class@1 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_class@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@3 as i_category, i_class@2 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)], projection=[ss_ext_sales_price@3, ss_net_profit@4, i_class@5, i_category@6] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_store_sk@1, ss_ext_sales_price@2, ss_net_profit@3, i_class@5, i_category@6] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_ext_sales_price@5, ss_net_profit@6] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([gross_margin@0, i_category@1, i_class@2, t_category@3, t_class@4, lochierarchy@5], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[gross_margin@0 as gross_margin, i_category@1 as i_category, i_class@2 as i_class, t_category@3 as t_category, t_class@4 as t_class, lochierarchy@5 as lochierarchy], aggr=[], ordering_mode=PartiallySorted([3, 4, 5]) + │ DistributedUnionExec: t0:[c0] t1:[c1] + │ ProjectionExec: expr=[gross_margin@0 as gross_margin, i_category@1 as i_category, CAST(i_class@2 AS Utf8) as i_class, 0 as t_category, 0 as t_class, 0 as lochierarchy] + │ ProjectionExec: expr=[CAST(sum(store_sales.ss_net_profit)@2 AS Float64) / CAST(sum(store_sales.ss_ext_sales_price)@3 AS Float64) as gross_margin, i_category@0 as i_category, i_class@1 as i_class] + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_class@1 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_class@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@3 as i_category, i_class@2 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)], projection=[ss_ext_sales_price@3, ss_net_profit@4, i_class@5, i_category@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_store_sk@1, ss_ext_sales_price@2, ss_net_profit@3, i_class@5, i_category@6] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_ext_sales_price@5, ss_net_profit@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[CAST(sum(results.ss_net_profit)@1 AS Float64) / CAST(sum(results.ss_ext_sales_price)@2 AS Float64) as gross_margin, i_category@0 as i_category, NULL as i_class, 0 as t_category, 1 as t_class, 1 as lochierarchy] + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category], aggr=[sum(results.ss_net_profit), sum(results.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@2 as i_category], aggr=[sum(results.ss_net_profit), sum(results.ss_ext_sales_price)] + │ ProjectionExec: expr=[sum(store_sales.ss_net_profit)@2 as ss_net_profit, sum(store_sales.ss_ext_sales_price)@3 as ss_ext_sales_price, i_category@0 as i_category] + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_class@1 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_category@0, i_class@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@3 as i_category, i_class@2 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)], projection=[ss_ext_sales_price@3, ss_net_profit@4, i_class@5, i_category@6] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_store_sk@1, ss_ext_sales_price@2, ss_net_profit@3, i_class@5, i_category@6] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_ext_sales_price@5, ss_net_profit@6] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@1 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@1 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@1 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_37() -> Result<()> { + let display = test_tpcds_query(37).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, i_current_price@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@0)], projection=[i_item_id@1, i_item_desc@2, i_current_price@3] + │ CoalescePartitionsExec │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([gross_margin@0, i_category@1, i_class@2, t_category@3, t_class@4, lochierarchy@5], 3), input_partitions=4 - │ AggregateExec: mode=Partial, gby=[gross_margin@0 as gross_margin, i_category@1 as i_category, i_class@2 as i_class, t_category@3 as t_category, t_class@4 as t_class, lochierarchy@5 as lochierarchy], aggr=[], ordering_mode=PartiallySorted([3]) - │ UnionExec - │ ProjectionExec: expr=[gross_margin@0 as gross_margin, CAST(i_category@1 AS Utf8) as i_category, i_class@2 as i_class, t_category@3 as t_category, t_class@4 as t_class, lochierarchy@5 as lochierarchy] - │ AggregateExec: mode=FinalPartitioned, gby=[gross_margin@0 as gross_margin, i_category@1 as i_category, i_class@2 as i_class, t_category@3 as t_category, t_class@4 as t_class, lochierarchy@5 as lochierarchy], aggr=[], ordering_mode=PartiallySorted([3]) - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([gross_margin@0, i_category@1, i_class@2, t_category@3, t_class@4, lochierarchy@5], 3), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[gross_margin@0 as gross_margin, i_category@1 as i_category, i_class@2 as i_class, t_category@3 as t_category, t_class@4 as t_class, lochierarchy@5 as lochierarchy], aggr=[], ordering_mode=PartiallySorted([3, 4, 5]) - │ UnionExec - │ ProjectionExec: expr=[gross_margin@0 as gross_margin, i_category@1 as i_category, CAST(i_class@2 AS Utf8) as i_class, 0 as t_category, 0 as t_class, 0 as lochierarchy] - │ ProjectionExec: expr=[CAST(sum(store_sales.ss_net_profit)@2 AS Float64) / CAST(sum(store_sales.ss_ext_sales_price)@3 AS Float64) as gross_margin, i_category@0 as i_category, i_class@1 as i_class] - │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_class@1 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([i_category@0, i_class@1], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[i_category@3 as i_category, i_class@2 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)], projection=[ss_ext_sales_price@3, ss_net_profit@4, i_class@5, i_category@6] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_store_sk@1, ss_ext_sales_price@2, ss_net_profit@3, i_class@5, i_category@6] - │ CoalescePartitionsExec - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_ext_sales_price@5, ss_net_profit@6] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=DynamicFilter [ empty ] - │ ProjectionExec: expr=[CAST(sum(results.ss_net_profit)@1 AS Float64) / CAST(sum(results.ss_ext_sales_price)@2 AS Float64) as gross_margin, i_category@0 as i_category, NULL as i_class, 0 as t_category, 1 as t_class, 1 as lochierarchy] - │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category], aggr=[sum(results.ss_net_profit), sum(results.ss_ext_sales_price)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([i_category@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[i_category@2 as i_category], aggr=[sum(results.ss_net_profit), sum(results.ss_ext_sales_price)] - │ ProjectionExec: expr=[sum(store_sales.ss_net_profit)@2 as ss_net_profit, sum(store_sales.ss_ext_sales_price)@3 as ss_ext_sales_price, i_category@0 as i_category] - │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_class@1 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([i_category@0, i_class@1], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[i_category@3 as i_category, i_class@2 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)], projection=[ss_ext_sales_price@3, ss_net_profit@4, i_class@5, i_category@6] - │ CoalescePartitionsExec - │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_store_sk@1, ss_ext_sales_price@2, ss_net_profit@3, i_class@5, i_category@6] - │ CoalescePartitionsExec - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_ext_sales_price@5, ss_net_profit@6] - │ CoalescePartitionsExec - │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=DynamicFilter [ empty ] - │ ProjectionExec: expr=[CAST(sum(results.ss_net_profit)@0 AS Float64) / CAST(sum(results.ss_ext_sales_price)@1 AS Float64) as gross_margin, NULL as i_category, NULL as i_class, 1 as t_category, 1 as t_class, 2 as lochierarchy] - │ AggregateExec: mode=Final, gby=[], aggr=[sum(results.ss_net_profit), sum(results.ss_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(inv_date_sk@4, d_date_sk@0)], projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[i_item_sk@1 as i_item_sk, i_item_id@2 as i_item_id, i_item_desc@3 as i_item_desc, i_current_price@4 as i_current_price, inv_date_sk@0 as inv_date_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(inv_item_sk@1, i_item_sk@0)], projection=[inv_date_sk@0, i_item_sk@2, i_item_id@3, i_item_desc@4, i_current_price@5] │ CoalescePartitionsExec - │ AggregateExec: mode=Partial, gby=[], aggr=[sum(results.ss_net_profit), sum(results.ss_ext_sales_price)] - │ ProjectionExec: expr=[sum(store_sales.ss_net_profit)@2 as ss_net_profit, sum(store_sales.ss_ext_sales_price)@3 as ss_ext_sales_price] - │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_class@1 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([i_category@0, i_class@1], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[i_category@3 as i_category, i_class@2 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)], projection=[ss_ext_sales_price@3, ss_net_profit@4, i_class@5, i_category@6] - │ CoalescePartitionsExec - │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_store_sk@1, ss_ext_sales_price@2, ss_net_profit@3, i_class@5, i_category@6] - │ CoalescePartitionsExec - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_ext_sales_price@5, ss_net_profit@6] - │ CoalescePartitionsExec - │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=DynamicFilter [ empty ] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@1 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 = 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@1 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 = 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@1 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] - └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 = 2001, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - └────────────────────────────────────────────────── - "#); - Ok(()) - } - #[tokio::test] - async fn test_tpcds_37() -> Result<()> { - let display = test_tpcds_query(37).await?; - assert_snapshot!(display, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST], fetch=100 - │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST], preserve_partitioning=[true] - │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price], aggr=[] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, i_current_price@2], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price], aggr=[] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@0)], projection=[i_item_id@1, i_item_desc@2, i_current_price@3] - │ CoalescePartitionsExec - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(inv_date_sk@4, d_date_sk@0)], projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3] - │ CoalescePartitionsExec - │ ProjectionExec: expr=[i_item_sk@1 as i_item_sk, i_item_id@2 as i_item_id, i_item_desc@3 as i_item_desc, i_current_price@4 as i_current_price, inv_date_sk@0 as inv_date_sk] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(inv_item_sk@1, i_item_sk@0)], projection=[inv_date_sk@0, i_item_sk@2, i_item_id@3, i_item_desc@4, i_current_price@5] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: i_current_price@3 >= Some(6800),4,2 AND i_current_price@3 <= Some(9800),4,2 AND i_manufact_id@4 IN (SET) ([677, 940, 694, 808]), projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@3 >= Some(6800),4,2 AND i_current_price@3 <= Some(9800),4,2 AND i_manufact_id@4 IN (SET) ([677, 940, 694, 808]) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(6800),4,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(9800),4,2 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 677 AND 677 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 940 AND 940 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 694 AND 694 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 808 AND 808 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (677, 694, 808, 940)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_date@1 >= 2000-02-01 AND d_date@1 <= 2000-04-01, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-02-01 AND d_date@1 <= 2000-04-01 AND DynamicFilter [ empty ], pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-01, required_guarantees=[] - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_current_price@3 >= Some(6800),4,2 AND i_current_price@3 <= Some(9800),4,2 AND i_manufact_id@4 IN (SET) ([677, 940, 694, 808]), projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@3 >= Some(6800),4,2 AND i_current_price@3 <= Some(9800),4,2 AND i_manufact_id@4 IN (SET) ([677, 940, 694, 808]) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(6800),4,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(9800),4,2 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 677 AND 677 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 940 AND 940 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 694 AND 694 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 808 AND 808 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (677, 694, 808, 940)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-02-01 AND d_date@1 <= 2000-04-01, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-02-01 AND d_date@1 <= 2000-04-01 AND DynamicFilter [ empty ], pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-01, required_guarantees=[] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ CoalesceBatchesExec: target_batch_size=8192 @@ -5050,157 +5071,160 @@ mod tests { │ SortPreservingMergeExec: [channel@0 ASC, return_rank@3 ASC, currency_rank@4 ASC, item@1 ASC], fetch=100 │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, return_rank@3 ASC, currency_rank@4 ASC, item@1 ASC], preserve_partitioning=[true] │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, item@1 as item, return_ratio@2 as return_ratio, return_rank@3 as return_rank, currency_rank@4 as currency_rank], aggr=[] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([channel@0, item@1, return_ratio@2, return_rank@3, currency_rank@4], 3), input_partitions=9 - │ AggregateExec: mode=Partial, gby=[channel@0 as channel, item@1 as item, return_ratio@2 as return_ratio, return_rank@3 as return_rank, currency_rank@4 as currency_rank], aggr=[], ordering_mode=PartiallySorted([0, 4]) - │ UnionExec - │ ProjectionExec: expr=[web as channel, item@0 as item, return_ratio@1 as return_ratio, return_rank@2 as return_rank, currency_rank@3 as currency_rank] - │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as return_rank, rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as currency_rank] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 <= 10 OR rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 <= 10 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1 - │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 as rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW] - │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] - │ SortExec: expr=[currency_ratio@2 ASC NULLS LAST], preserve_partitioning=[false] - │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] - │ SortPreservingMergeExec: [return_ratio@1 ASC NULLS LAST] - │ SortExec: expr=[return_ratio@1 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[ws_item_sk@0 as item, CAST(sum(coalesce(wr.wr_return_quantity,Int64(0)))@1 AS Decimal128(15, 4)) / CAST(sum(coalesce(ws.ws_quantity,Int64(0)))@2 AS Decimal128(15, 4)) as return_ratio, CAST(sum(coalesce(wr.wr_return_amt,Int64(0)))@3 AS Decimal128(15, 4)) / CAST(sum(coalesce(ws.ws_net_paid,Int64(0)))@4 AS Decimal128(15, 4)) as currency_ratio] - │ AggregateExec: mode=FinalPartitioned, gby=[ws_item_sk@0 as ws_item_sk], aggr=[sum(coalesce(wr.wr_return_quantity,Int64(0))), sum(coalesce(ws.ws_quantity,Int64(0))), sum(coalesce(wr.wr_return_amt,Int64(0))), sum(coalesce(ws.ws_net_paid,Int64(0)))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_item_sk@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[ws_item_sk@0 as ws_item_sk], aggr=[sum(coalesce(wr.wr_return_quantity,Int64(0))), sum(coalesce(ws.ws_quantity,Int64(0))), sum(coalesce(wr.wr_return_amt,Int64(0))), sum(coalesce(ws.ws_net_paid,Int64(0)))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_quantity@4, ws_net_paid@5, wr_return_quantity@6, wr_return_amt@7] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: wr_return_amt@5 > Some(1000000),7,2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(ws_order_number@2, wr_order_number@1), (ws_item_sk@1, wr_item_sk@0)], projection=[ws_sold_date_sk@0, ws_item_sk@1, ws_quantity@3, ws_net_paid@4, wr_return_quantity@7, wr_return_amt@8] - │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ ProjectionExec: expr=[catalog as channel, item@0 as item, return_ratio@1 as return_ratio, return_rank@2 as return_rank, currency_rank@3 as currency_rank] - │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as return_rank, rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as currency_rank] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 <= 10 OR rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 <= 10 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1 - │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 as rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW] - │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] - │ SortExec: expr=[currency_ratio@2 ASC NULLS LAST], preserve_partitioning=[false] - │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] - │ SortPreservingMergeExec: [return_ratio@1 ASC NULLS LAST] - │ SortExec: expr=[return_ratio@1 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[cs_item_sk@0 as item, CAST(sum(coalesce(cr.cr_return_quantity,Int64(0)))@1 AS Decimal128(15, 4)) / CAST(sum(coalesce(cs.cs_quantity,Int64(0)))@2 AS Decimal128(15, 4)) as return_ratio, CAST(sum(coalesce(cr.cr_return_amount,Int64(0)))@3 AS Decimal128(15, 4)) / CAST(sum(coalesce(cs.cs_net_paid,Int64(0)))@4 AS Decimal128(15, 4)) as currency_ratio] - │ AggregateExec: mode=FinalPartitioned, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(coalesce(cr.cr_return_quantity,Int64(0))), sum(coalesce(cs.cs_quantity,Int64(0))), sum(coalesce(cr.cr_return_amount,Int64(0))), sum(coalesce(cs.cs_net_paid,Int64(0)))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_item_sk@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(coalesce(cr.cr_return_quantity,Int64(0))), sum(coalesce(cs.cs_quantity,Int64(0))), sum(coalesce(cr.cr_return_amount,Int64(0))), sum(coalesce(cs.cs_net_paid,Int64(0)))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_item_sk@3, cs_quantity@4, cs_net_paid@5, cr_return_quantity@6, cr_return_amount@7] - │ CoalescePartitionsExec - │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: cr_return_amount@5 > Some(1000000),7,2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(cs_order_number@2, cr_order_number@1), (cs_item_sk@1, cr_item_sk@0)], projection=[cs_sold_date_sk@0, cs_item_sk@1, cs_quantity@3, cs_net_paid@4, cr_return_quantity@7, cr_return_amount@8] - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ ProjectionExec: expr=[store as channel, item@0 as item, return_ratio@1 as return_ratio, return_rank@2 as return_rank, currency_rank@3 as currency_rank] - │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as return_rank, rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as currency_rank] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 <= 10 OR rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 <= 10 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1 - │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 as rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW] - │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] - │ SortExec: expr=[currency_ratio@2 ASC NULLS LAST], preserve_partitioning=[false] - │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] - │ SortPreservingMergeExec: [return_ratio@1 ASC NULLS LAST] - │ SortExec: expr=[return_ratio@1 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[ss_item_sk@0 as item, CAST(sum(coalesce(sr.sr_return_quantity,Int64(0)))@1 AS Decimal128(15, 4)) / CAST(sum(coalesce(sts.ss_quantity,Int64(0)))@2 AS Decimal128(15, 4)) as return_ratio, CAST(sum(coalesce(sr.sr_return_amt,Int64(0)))@3 AS Decimal128(15, 4)) / CAST(sum(coalesce(sts.ss_net_paid,Int64(0)))@4 AS Decimal128(15, 4)) as currency_ratio] - │ AggregateExec: mode=FinalPartitioned, gby=[ss_item_sk@0 as ss_item_sk], aggr=[sum(coalesce(sr.sr_return_quantity,Int64(0))), sum(coalesce(sts.ss_quantity,Int64(0))), sum(coalesce(sr.sr_return_amt,Int64(0))), sum(coalesce(sts.ss_net_paid,Int64(0)))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_item_sk@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[ss_item_sk@0 as ss_item_sk], aggr=[sum(coalesce(sr.sr_return_quantity,Int64(0))), sum(coalesce(sts.ss_quantity,Int64(0))), sum(coalesce(sr.sr_return_amt,Int64(0))), sum(coalesce(sts.ss_net_paid,Int64(0)))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_quantity@4, ss_net_paid@5, sr_return_quantity@6, sr_return_amt@7] - │ CoalescePartitionsExec - │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: sr_return_amt@5 > Some(1000000),7,2 - │ ProjectionExec: expr=[ss_sold_date_sk@2 as ss_sold_date_sk, ss_item_sk@3 as ss_item_sk, ss_quantity@4 as ss_quantity, ss_net_paid@5 as ss_net_paid, sr_return_quantity@0 as sr_return_quantity, sr_return_amt@1 as sr_return_amt] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(sr_ticket_number@1, ss_ticket_number@2), (sr_item_sk@0, ss_item_sk@1)], projection=[sr_return_quantity@2, sr_return_amt@3, ss_sold_date_sk@4, ss_item_sk@5, ss_quantity@7, ss_net_paid@8] - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 12, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_order_number@2, ws_item_sk@1], 3), input_partitions=3 - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: ws_net_profit@5 > Some(100),7,2 AND ws_net_paid@4 > Some(0),7,2 AND ws_quantity@3 > 0, projection=[ws_sold_date_sk@0, ws_item_sk@1, ws_order_number@2, ws_quantity@3, ws_net_paid@4] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_net_paid, ws_net_profit], file_type=parquet, predicate=ws_net_profit@5 > Some(100),7,2 AND ws_net_paid@4 > Some(0),7,2 AND ws_quantity@3 > 0, pruning_predicate=ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 > Some(100),7,2 AND ws_net_paid_null_count@4 != row_count@2 AND ws_net_paid_max@3 > Some(0),7,2 AND ws_quantity_null_count@6 != row_count@2 AND ws_quantity_max@5 > 0, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([wr_order_number@1, wr_item_sk@0], 3), input_partitions=3 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 12, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] - └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_order_number@2, cs_item_sk@1], 3), input_partitions=3 - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: cs_net_profit@5 > Some(100),7,2 AND cs_net_paid@4 > Some(0),7,2 AND cs_quantity@3 > 0, projection=[cs_sold_date_sk@0, cs_item_sk@1, cs_order_number@2, cs_quantity@3, cs_net_paid@4] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_net_paid, cs_net_profit], file_type=parquet, predicate=cs_net_profit@5 > Some(100),7,2 AND cs_net_paid@4 > Some(0),7,2 AND cs_quantity@3 > 0, pruning_predicate=cs_net_profit_null_count@1 != row_count@2 AND cs_net_profit_max@0 > Some(100),7,2 AND cs_net_paid_null_count@4 != row_count@2 AND cs_net_paid_max@3 > Some(0),7,2 AND cs_quantity_null_count@6 != row_count@2 AND cs_quantity_max@5 > 0, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cr_order_number@1, cr_item_sk@0], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 12, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] - └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_ticket_number@2, ss_item_sk@1], 3), input_partitions=3 - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: ss_net_profit@5 > Some(100),6,2 AND ss_net_paid@4 > Some(0),7,2 AND ss_quantity@3 > 0, projection=[ss_sold_date_sk@0, ss_item_sk@1, ss_ticket_number@2, ss_quantity@3, ss_net_paid@4] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_net_paid, ss_net_profit], file_type=parquet, predicate=ss_net_profit@5 > Some(100),6,2 AND ss_net_paid@4 > Some(0),7,2 AND ss_quantity@3 > 0, pruning_predicate=ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 > Some(100),6,2 AND ss_net_paid_null_count@4 != row_count@2 AND ss_net_paid_max@3 > Some(0),7,2 AND ss_quantity_null_count@6 != row_count@2 AND ss_quantity_max@5 > 0, required_guarantees=[] - └────────────────────────────────────────────────── + │ RepartitionExec: partitioning=Hash([channel@0, item@1, return_ratio@2, return_rank@3, currency_rank@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[channel@0 as channel, item@1 as item, return_ratio@2 as return_ratio, return_rank@3 as return_rank, currency_rank@4 as currency_rank], aggr=[], ordering_mode=PartiallySorted([0, 4]) + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ ProjectionExec: expr=[web as channel, item@0 as item, return_ratio@1 as return_ratio, return_rank@2 as return_rank, currency_rank@3 as currency_rank] + │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as return_rank, rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as currency_rank] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 <= 10 OR rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 <= 10 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1 + │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 as rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[currency_ratio@2 ASC NULLS LAST], preserve_partitioning=[false] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortPreservingMergeExec: [return_ratio@1 ASC NULLS LAST] + │ SortExec: expr=[return_ratio@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[ws_item_sk@0 as item, CAST(sum(coalesce(wr.wr_return_quantity,Int64(0)))@1 AS Decimal128(15, 4)) / CAST(sum(coalesce(ws.ws_quantity,Int64(0)))@2 AS Decimal128(15, 4)) as return_ratio, CAST(sum(coalesce(wr.wr_return_amt,Int64(0)))@3 AS Decimal128(15, 4)) / CAST(sum(coalesce(ws.ws_net_paid,Int64(0)))@4 AS Decimal128(15, 4)) as currency_ratio] + │ AggregateExec: mode=FinalPartitioned, gby=[ws_item_sk@0 as ws_item_sk], aggr=[sum(coalesce(wr.wr_return_quantity,Int64(0))), sum(coalesce(ws.ws_quantity,Int64(0))), sum(coalesce(wr.wr_return_amt,Int64(0))), sum(coalesce(ws.ws_net_paid,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_item_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ws_item_sk@0 as ws_item_sk], aggr=[sum(coalesce(wr.wr_return_quantity,Int64(0))), sum(coalesce(ws.ws_quantity,Int64(0))), sum(coalesce(wr.wr_return_amt,Int64(0))), sum(coalesce(ws.ws_net_paid,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_quantity@4, ws_net_paid@5, wr_return_quantity@6, wr_return_amt@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: wr_return_amt@5 > Some(1000000),7,2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(ws_order_number@2, wr_order_number@1), (ws_item_sk@1, wr_item_sk@0)], projection=[ws_sold_date_sk@0, ws_item_sk@1, ws_quantity@3, ws_net_paid@4, wr_return_quantity@7, wr_return_amt@8] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[catalog as channel, item@0 as item, return_ratio@1 as return_ratio, return_rank@2 as return_rank, currency_rank@3 as currency_rank] + │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as return_rank, rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as currency_rank] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 <= 10 OR rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 <= 10 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1 + │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 as rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[currency_ratio@2 ASC NULLS LAST], preserve_partitioning=[false] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortPreservingMergeExec: [return_ratio@1 ASC NULLS LAST] + │ SortExec: expr=[return_ratio@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[cs_item_sk@0 as item, CAST(sum(coalesce(cr.cr_return_quantity,Int64(0)))@1 AS Decimal128(15, 4)) / CAST(sum(coalesce(cs.cs_quantity,Int64(0)))@2 AS Decimal128(15, 4)) as return_ratio, CAST(sum(coalesce(cr.cr_return_amount,Int64(0)))@3 AS Decimal128(15, 4)) / CAST(sum(coalesce(cs.cs_net_paid,Int64(0)))@4 AS Decimal128(15, 4)) as currency_ratio] + │ AggregateExec: mode=FinalPartitioned, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(coalesce(cr.cr_return_quantity,Int64(0))), sum(coalesce(cs.cs_quantity,Int64(0))), sum(coalesce(cr.cr_return_amount,Int64(0))), sum(coalesce(cs.cs_net_paid,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(coalesce(cr.cr_return_quantity,Int64(0))), sum(coalesce(cs.cs_quantity,Int64(0))), sum(coalesce(cr.cr_return_amount,Int64(0))), sum(coalesce(cs.cs_net_paid,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_item_sk@3, cs_quantity@4, cs_net_paid@5, cr_return_quantity@6, cr_return_amount@7] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cr_return_amount@5 > Some(1000000),7,2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(cs_order_number@2, cr_order_number@1), (cs_item_sk@1, cr_item_sk@0)], projection=[cs_sold_date_sk@0, cs_item_sk@1, cs_quantity@3, cs_net_paid@4, cr_return_quantity@7, cr_return_amount@8] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[store as channel, item@0 as item, return_ratio@1 as return_ratio, return_rank@2 as return_rank, currency_rank@3 as currency_rank] + │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as return_rank, rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as currency_rank] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 <= 10 OR rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 <= 10 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1 + │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 as rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[currency_ratio@2 ASC NULLS LAST], preserve_partitioning=[false] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortPreservingMergeExec: [return_ratio@1 ASC NULLS LAST] + │ SortExec: expr=[return_ratio@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[ss_item_sk@0 as item, CAST(sum(coalesce(sr.sr_return_quantity,Int64(0)))@1 AS Decimal128(15, 4)) / CAST(sum(coalesce(sts.ss_quantity,Int64(0)))@2 AS Decimal128(15, 4)) as return_ratio, CAST(sum(coalesce(sr.sr_return_amt,Int64(0)))@3 AS Decimal128(15, 4)) / CAST(sum(coalesce(sts.ss_net_paid,Int64(0)))@4 AS Decimal128(15, 4)) as currency_ratio] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_item_sk@0 as ss_item_sk], aggr=[sum(coalesce(sr.sr_return_quantity,Int64(0))), sum(coalesce(sts.ss_quantity,Int64(0))), sum(coalesce(sr.sr_return_amt,Int64(0))), sum(coalesce(sts.ss_net_paid,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_item_sk@0 as ss_item_sk], aggr=[sum(coalesce(sr.sr_return_quantity,Int64(0))), sum(coalesce(sts.ss_quantity,Int64(0))), sum(coalesce(sr.sr_return_amt,Int64(0))), sum(coalesce(sts.ss_net_paid,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_quantity@4, ss_net_paid@5, sr_return_quantity@6, sr_return_amt@7] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: sr_return_amt@5 > Some(1000000),7,2 + │ ProjectionExec: expr=[ss_sold_date_sk@2 as ss_sold_date_sk, ss_item_sk@3 as ss_item_sk, ss_quantity@4 as ss_quantity, ss_net_paid@5 as ss_net_paid, sr_return_quantity@0 as sr_return_quantity, sr_return_amt@1 as sr_return_amt] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(sr_ticket_number@1, ss_ticket_number@2), (sr_item_sk@0, ss_item_sk@1)], projection=[sr_return_quantity@2, sr_return_amt@3, ss_sold_date_sk@4, ss_item_sk@5, ss_quantity@7, ss_net_paid@8] + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 12, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_order_number@2, ws_item_sk@1], 3), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ws_net_profit@5 > Some(100),7,2 AND ws_net_paid@4 > Some(0),7,2 AND ws_quantity@3 > 0, projection=[ws_sold_date_sk@0, ws_item_sk@1, ws_order_number@2, ws_quantity@3, ws_net_paid@4] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_net_paid, ws_net_profit], file_type=parquet, predicate=ws_net_profit@5 > Some(100),7,2 AND ws_net_paid@4 > Some(0),7,2 AND ws_quantity@3 > 0, pruning_predicate=ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 > Some(100),7,2 AND ws_net_paid_null_count@4 != row_count@2 AND ws_net_paid_max@3 > Some(0),7,2 AND ws_quantity_null_count@6 != row_count@2 AND ws_quantity_max@5 > 0, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([wr_order_number@1, wr_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 12, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_order_number@2, cs_item_sk@1], 3), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cs_net_profit@5 > Some(100),7,2 AND cs_net_paid@4 > Some(0),7,2 AND cs_quantity@3 > 0, projection=[cs_sold_date_sk@0, cs_item_sk@1, cs_order_number@2, cs_quantity@3, cs_net_paid@4] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_net_paid, cs_net_profit], file_type=parquet, predicate=cs_net_profit@5 > Some(100),7,2 AND cs_net_paid@4 > Some(0),7,2 AND cs_quantity@3 > 0, pruning_predicate=cs_net_profit_null_count@1 != row_count@2 AND cs_net_profit_max@0 > Some(100),7,2 AND cs_net_paid_null_count@4 != row_count@2 AND cs_net_paid_max@3 > Some(0),7,2 AND cs_quantity_null_count@6 != row_count@2 AND cs_quantity_max@5 > 0, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cr_order_number@1, cr_item_sk@0], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 12, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_ticket_number@2, ss_item_sk@1], 3), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_net_profit@5 > Some(100),6,2 AND ss_net_paid@4 > Some(0),7,2 AND ss_quantity@3 > 0, projection=[ss_sold_date_sk@0, ss_item_sk@1, ss_ticket_number@2, ss_quantity@3, ss_net_paid@4] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_net_paid, ss_net_profit], file_type=parquet, predicate=ss_net_profit@5 > Some(100),6,2 AND ss_net_paid@4 > Some(0),7,2 AND ss_quantity@3 > 0, pruning_predicate=ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 > Some(100),6,2 AND ss_net_paid_null_count@4 != row_count@2 AND ss_net_paid_max@3 > Some(0),7,2 AND ss_quantity_null_count@6 != row_count@2 AND ss_quantity_max@5 > 0, required_guarantees=[] + └────────────────────────────────────────────────── "#); Ok(()) } @@ -5483,7 +5507,7 @@ mod tests { │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, item_sk@2)], projection=[sold_date_sk@1, customer_sk@2] │ CoalescePartitionsExec │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ UnionExec + │ DistributedUnionExec: t0:[c0, c1] │ ProjectionExec: expr=[cs_sold_date_sk@0 as sold_date_sk, cs_bill_customer_sk@1 as customer_sk, cs_item_sk@2 as item_sk] │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk], file_type=parquet │ ProjectionExec: expr=[ws_sold_date_sk@0 as sold_date_sk, ws_bill_customer_sk@2 as customer_sk, ws_item_sk@1 as item_sk] @@ -7250,25 +7274,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(sold_item_sk@1, i_item_sk@0)], projection=[ext_price@0, time_sk@2, i_brand_id@4, i_brand@5] │ CoalescePartitionsExec - │ UnionExec - │ ProjectionExec: expr=[ws_ext_sales_price@2 as ext_price, ws_item_sk@1 as sold_item_sk, ws_sold_time_sk@0 as time_sk] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_sold_time_sk@3, ws_item_sk@4, ws_ext_sales_price@5] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_sold_time_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ ProjectionExec: expr=[cs_ext_sales_price@2 as ext_price, cs_item_sk@1 as sold_item_sk, cs_sold_time_sk@0 as time_sk] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_sold_time_sk@3, cs_item_sk@4, cs_ext_sales_price@5] - │ CoalescePartitionsExec - │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_sold_time_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] - │ ProjectionExec: expr=[ss_ext_sales_price@2 as ext_price, ss_item_sk@1 as sold_item_sk, ss_sold_time_sk@0 as time_sk] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_sold_time_sk@3, ss_item_sk@4, ss_ext_sales_price@5] - │ CoalescePartitionsExec - │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_sold_time_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ [Stage 5] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: i_manager_id@3 = 1, projection=[i_item_sk@0, i_brand_id@1, i_brand@2] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 @@ -7282,30 +7288,51 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute, t_meal_time], file_type=parquet, predicate=t_meal_time@3 = breakfast OR t_meal_time@3 = dinner, pruning_predicate=t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= breakfast AND breakfast <= t_meal_time_max@1 OR t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= dinner AND dinner <= t_meal_time_max@1, required_guarantees=[t_meal_time in (breakfast, dinner)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1999, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 11 AND d_year@1 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1999, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 11 AND d_year@1 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1999, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 11 AND d_year@1 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] - └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ ProjectionExec: expr=[ws_ext_sales_price@2 as ext_price, ws_item_sk@1 as sold_item_sk, ws_sold_time_sk@0 as time_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_sold_time_sk@3, ws_item_sk@4, ws_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_sold_time_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[cs_ext_sales_price@2 as ext_price, cs_item_sk@1 as sold_item_sk, cs_sold_time_sk@0 as time_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_sold_time_sk@3, cs_item_sk@4, cs_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_sold_time_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ss_ext_sales_price@2 as ext_price, ss_item_sk@1 as sold_item_sk, ss_sold_time_sk@0 as time_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_sold_time_sk@3, ss_item_sk@4, ss_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_sold_time_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1999, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 11 AND d_year@1 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1999, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 11 AND d_year@1 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1999, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 11 AND d_year@1 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + └────────────────────────────────────────────────── "); Ok(()) } @@ -7668,524 +7695,545 @@ mod tests { │ CoalescePartitionsExec │ ProjectionExec: expr=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sum(sales_detail.sales_cnt)@5 as sales_cnt, sum(sales_detail.sales_amt)@6 as sales_amt] │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id], aggr=[sum(sales_detail.sales_cnt), sum(sales_detail.sales_amt)], ordering_mode=PartiallySorted([0]) - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id], aggr=[sum(sales_detail.sales_cnt), sum(sales_detail.sales_amt)], ordering_mode=PartiallySorted([0]) - │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sales_cnt@5 as sales_cnt, sales_amt@6 as sales_amt], aggr=[], ordering_mode=PartiallySorted([0]) - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, sales_cnt@5, sales_amt@6], 3), input_partitions=9 - │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sales_cnt@5 as sales_cnt, sales_amt@6 as sales_amt], aggr=[], ordering_mode=PartiallySorted([0]) - │ UnionExec - │ ProjectionExec: expr=[d_year@6 as d_year, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, cs_quantity@0 - coalesce(cr_return_quantity@7, 0) as sales_cnt, cs_ext_sales_price@1 - coalesce(CAST(cr_return_amount@8 AS Decimal128(30, 15)), Some(0),30,15) as sales_amt] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(cs_order_number@1, cr_order_number@1), (cs_item_sk@0, cr_item_sk@0)], projection=[cs_quantity@2, cs_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, cr_return_quantity@11, cr_return_amount@12] - │ CoalescePartitionsExec - │ ProjectionExec: expr=[cs_item_sk@1 as cs_item_sk, cs_order_number@2 as cs_order_number, cs_quantity@3 as cs_quantity, cs_ext_sales_price@4 as cs_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@0)], projection=[d_year@1, cs_item_sk@4, cs_order_number@5, cs_quantity@6, cs_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ ProjectionExec: expr=[cs_sold_date_sk@4 as cs_sold_date_sk, cs_item_sk@5 as cs_item_sk, cs_order_number@6 as cs_order_number, cs_quantity@7 as cs_quantity, cs_ext_sales_price@8 as cs_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, cs_sold_date_sk@5, cs_item_sk@6, cs_order_number@7, cs_quantity@8, cs_ext_sales_price@9] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet - │ ProjectionExec: expr=[d_year@6 as d_year, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, ss_quantity@0 - coalesce(sr_return_quantity@7, 0) as sales_cnt, ss_ext_sales_price@1 - coalesce(CAST(sr_return_amt@8 AS Decimal128(30, 15)), Some(0),30,15) as sales_amt] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(ss_ticket_number@1, sr_ticket_number@1), (ss_item_sk@0, sr_item_sk@0)], projection=[ss_quantity@2, ss_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, sr_return_quantity@11, sr_return_amt@12] - │ CoalescePartitionsExec - │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_ticket_number@2 as ss_ticket_number, ss_quantity@3 as ss_quantity, ss_ext_sales_price@4 as ss_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_ticket_number@5, ss_quantity@6, ss_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] - │ CoalescePartitionsExec - │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ ProjectionExec: expr=[ss_sold_date_sk@4 as ss_sold_date_sk, ss_item_sk@5 as ss_item_sk, ss_ticket_number@6 as ss_ticket_number, ss_quantity@7 as ss_quantity, ss_ext_sales_price@8 as ss_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, ss_sold_date_sk@5, ss_item_sk@6, ss_ticket_number@7, ss_quantity@8, ss_ext_sales_price@9] - │ CoalescePartitionsExec - │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet - │ ProjectionExec: expr=[d_year@6 as d_year, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, ws_quantity@0 - coalesce(wr_return_quantity@7, 0) as sales_cnt, ws_ext_sales_price@1 - coalesce(CAST(wr_return_amt@8 AS Decimal128(30, 15)), Some(0),30,15) as sales_amt] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(ws_order_number@1, wr_order_number@1), (ws_item_sk@0, wr_item_sk@0)], projection=[ws_quantity@2, ws_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, wr_return_quantity@11, wr_return_amt@12] - │ CoalescePartitionsExec - │ ProjectionExec: expr=[ws_item_sk@1 as ws_item_sk, ws_order_number@2 as ws_order_number, ws_quantity@3 as ws_quantity, ws_ext_sales_price@4 as ws_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@0)], projection=[d_year@1, ws_item_sk@4, ws_order_number@5, ws_quantity@6, ws_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] - │ CoalescePartitionsExec - │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ ProjectionExec: expr=[ws_sold_date_sk@4 as ws_sold_date_sk, ws_item_sk@5 as ws_item_sk, ws_order_number@6 as ws_order_number, ws_quantity@7 as ws_quantity, ws_ext_sales_price@8 as ws_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, ws_sold_date_sk@5, ws_item_sk@6, ws_order_number@7, ws_quantity@8, ws_ext_sales_price@9] - │ CoalescePartitionsExec - │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ ProjectionExec: expr=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sum(sales_detail.sales_cnt)@5 as sales_cnt, sum(sales_detail.sales_amt)@6 as sales_amt] │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id], aggr=[sum(sales_detail.sales_cnt), sum(sales_detail.sales_amt)], ordering_mode=PartiallySorted([0]) - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id], aggr=[sum(sales_detail.sales_cnt), sum(sales_detail.sales_amt)], ordering_mode=PartiallySorted([0]) - │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sales_cnt@5 as sales_cnt, sales_amt@6 as sales_amt], aggr=[], ordering_mode=PartiallySorted([0]) - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, sales_cnt@5, sales_amt@6], 3), input_partitions=9 - │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sales_cnt@5 as sales_cnt, sales_amt@6 as sales_amt], aggr=[], ordering_mode=PartiallySorted([0]) - │ UnionExec - │ ProjectionExec: expr=[d_year@6 as d_year, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, cs_quantity@0 - coalesce(cr_return_quantity@7, 0) as sales_cnt, cs_ext_sales_price@1 - coalesce(CAST(cr_return_amount@8 AS Decimal128(30, 15)), Some(0),30,15) as sales_amt] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(cs_order_number@1, cr_order_number@1), (cs_item_sk@0, cr_item_sk@0)], projection=[cs_quantity@2, cs_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, cr_return_quantity@11, cr_return_amount@12] - │ CoalescePartitionsExec - │ ProjectionExec: expr=[cs_item_sk@1 as cs_item_sk, cs_order_number@2 as cs_order_number, cs_quantity@3 as cs_quantity, cs_ext_sales_price@4 as cs_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@0)], projection=[d_year@1, cs_item_sk@4, cs_order_number@5, cs_quantity@6, cs_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] - │ CoalescePartitionsExec - │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ ProjectionExec: expr=[cs_sold_date_sk@4 as cs_sold_date_sk, cs_item_sk@5 as cs_item_sk, cs_order_number@6 as cs_order_number, cs_quantity@7 as cs_quantity, cs_ext_sales_price@8 as cs_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, cs_sold_date_sk@5, cs_item_sk@6, cs_order_number@7, cs_quantity@8, cs_ext_sales_price@9] - │ CoalescePartitionsExec - │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet - │ ProjectionExec: expr=[d_year@6 as d_year, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, ss_quantity@0 - coalesce(sr_return_quantity@7, 0) as sales_cnt, ss_ext_sales_price@1 - coalesce(CAST(sr_return_amt@8 AS Decimal128(30, 15)), Some(0),30,15) as sales_amt] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(ss_ticket_number@1, sr_ticket_number@1), (ss_item_sk@0, sr_item_sk@0)], projection=[ss_quantity@2, ss_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, sr_return_quantity@11, sr_return_amt@12] - │ CoalescePartitionsExec - │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_ticket_number@2 as ss_ticket_number, ss_quantity@3 as ss_quantity, ss_ext_sales_price@4 as ss_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_ticket_number@5, ss_quantity@6, ss_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] - │ CoalescePartitionsExec - │ [Stage 9] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ ProjectionExec: expr=[ss_sold_date_sk@4 as ss_sold_date_sk, ss_item_sk@5 as ss_item_sk, ss_ticket_number@6 as ss_ticket_number, ss_quantity@7 as ss_quantity, ss_ext_sales_price@8 as ss_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, ss_sold_date_sk@5, ss_item_sk@6, ss_ticket_number@7, ss_quantity@8, ss_ext_sales_price@9] - │ CoalescePartitionsExec - │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet - │ ProjectionExec: expr=[d_year@6 as d_year, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, ws_quantity@0 - coalesce(wr_return_quantity@7, 0) as sales_cnt, ws_ext_sales_price@1 - coalesce(CAST(wr_return_amt@8 AS Decimal128(30, 15)), Some(0),30,15) as sales_amt] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(ws_order_number@1, wr_order_number@1), (ws_item_sk@0, wr_item_sk@0)], projection=[ws_quantity@2, ws_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, wr_return_quantity@11, wr_return_amt@12] - │ CoalescePartitionsExec - │ ProjectionExec: expr=[ws_item_sk@1 as ws_item_sk, ws_order_number@2 as ws_order_number, ws_quantity@3 as ws_quantity, ws_ext_sales_price@4 as ws_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@0)], projection=[d_year@1, ws_item_sk@4, ws_order_number@5, ws_quantity@6, ws_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] - │ CoalescePartitionsExec - │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ ProjectionExec: expr=[ws_sold_date_sk@4 as ws_sold_date_sk, ws_item_sk@5 as ws_item_sk, ws_order_number@6 as ws_order_number, ws_quantity@7 as ws_quantity, ws_ext_sales_price@8 as ws_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, ws_sold_date_sk@5, ws_item_sk@6, ws_order_number@7, ws_quantity@8, ws_ext_sales_price@9] - │ CoalescePartitionsExec - │ [Stage 12] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet + │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 = 2002 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@4 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 = 2002 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@4 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 = 2002 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] - └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@4 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 = 2001 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@4 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - └────────────────────────────────────────────────── - ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 = 2001 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - └────────────────────────────────────────────────── - ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@4 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - └────────────────────────────────────────────────── - ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_year@1 = 2001 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] - └────────────────────────────────────────────────── - ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@4 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] - └────────────────────────────────────────────────── - "); - Ok(()) - } - #[tokio::test] - async fn test_tpcds_76() -> Result<()> { - let display = test_tpcds_query(76).await?; - assert_snapshot!(display, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [channel@0 ASC, col_name@1 ASC, d_year@2 ASC, d_qoy@3 ASC, i_category@4 ASC], fetch=100 - │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, col_name@1 ASC, d_year@2 ASC, d_qoy@3 ASC, i_category@4 ASC], preserve_partitioning=[true] - │ ProjectionExec: expr=[channel@0 as channel, col_name@1 as col_name, d_year@2 as d_year, d_qoy@3 as d_qoy, i_category@4 as i_category, count(Int64(1))@5 as sales_cnt, sum(foo.ext_sales_price)@6 as sales_amt] - │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, col_name@1 as col_name, d_year@2 as d_year, d_qoy@3 as d_qoy, i_category@4 as i_category], aggr=[count(Int64(1)), sum(foo.ext_sales_price)] - │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - └────────────────────────────────────────────────── - ┌───── Stage 13 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([channel@0, col_name@1, d_year@2, d_qoy@3, i_category@4], 3), input_partitions=9 - │ AggregateExec: mode=Partial, gby=[channel@0 as channel, col_name@1 as col_name, d_year@2 as d_year, d_qoy@3 as d_qoy, i_category@4 as i_category], aggr=[count(Int64(1)), sum(foo.ext_sales_price)], ordering_mode=PartiallySorted([0, 1]) - │ UnionExec - │ ProjectionExec: expr=[store as channel, ss_store_sk as col_name, d_year@4 as d_year, d_qoy@5 as d_qoy, i_category@2 as i_category, ss_ext_sales_price@1 as ext_sales_price] - │ ProjectionExec: expr=[ss_sold_date_sk@3 as ss_sold_date_sk, ss_ext_sales_price@4 as ss_ext_sales_price, i_category@5 as i_category, d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@0)], projection=[d_date_sk@0, d_year@1, d_qoy@2, ss_sold_date_sk@4, ss_ext_sales_price@5, i_category@6] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ ProjectionExec: expr=[web as channel, ws_ship_customer_sk as col_name, d_year@4 as d_year, d_qoy@5 as d_qoy, i_category@2 as i_category, ws_ext_sales_price@1 as ext_sales_price] - │ ProjectionExec: expr=[ws_sold_date_sk@3 as ws_sold_date_sk, ws_ext_sales_price@4 as ws_ext_sales_price, i_category@5 as i_category, d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ws_sold_date_sk@0)], projection=[d_date_sk@0, d_year@1, d_qoy@2, ws_sold_date_sk@4, ws_ext_sales_price@5, i_category@6] - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ ProjectionExec: expr=[catalog as channel, cs_ship_addr_sk as col_name, d_year@4 as d_year, d_qoy@5 as d_qoy, i_category@2 as i_category, cs_ext_sales_price@1 as ext_sales_price] - │ ProjectionExec: expr=[cs_sold_date_sk@3 as cs_sold_date_sk, cs_ext_sales_price@4 as cs_ext_sales_price, i_category@5 as i_category, d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, cs_sold_date_sk@0)], projection=[d_date_sk@0, d_year@1, d_qoy@2, cs_sold_date_sk@4, cs_ext_sales_price@5, i_category@6] - │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=Hash([d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id], aggr=[sum(sales_detail.sales_cnt), sum(sales_detail.sales_amt)], ordering_mode=PartiallySorted([0]) + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sales_cnt@5 as sales_cnt, sales_amt@6 as sales_amt], aggr=[], ordering_mode=PartiallySorted([0]) + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 6), input_partitions=3 - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_sold_date_sk@0], 6), input_partitions=3 - │ ProjectionExec: expr=[ss_sold_date_sk@1 as ss_sold_date_sk, ss_ext_sales_price@2 as ss_ext_sales_price, i_category@0 as i_category] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_category@1, ss_sold_date_sk@2, ss_ext_sales_price@4] - │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=Hash([d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, sales_cnt@5, sales_amt@6], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sales_cnt@5 as sales_cnt, sales_amt@6 as sales_amt], aggr=[], ordering_mode=PartiallySorted([0]) + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ ProjectionExec: expr=[d_year@6 as d_year, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, cs_quantity@0 - coalesce(cr_return_quantity@7, 0) as sales_cnt, cs_ext_sales_price@1 - coalesce(CAST(cr_return_amount@8 AS Decimal128(30, 15)), Some(0),30,15) as sales_amt] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(cs_order_number@1, cr_order_number@1), (cs_item_sk@0, cr_item_sk@0)], projection=[cs_quantity@2, cs_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, cr_return_quantity@11, cr_return_amount@12] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[cs_item_sk@1 as cs_item_sk, cs_order_number@2 as cs_order_number, cs_quantity@3 as cs_quantity, cs_ext_sales_price@4 as cs_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@0)], projection=[d_year@1, cs_item_sk@4, cs_order_number@5, cs_quantity@6, cs_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@4 as cs_sold_date_sk, cs_item_sk@5 as cs_item_sk, cs_order_number@6 as cs_order_number, cs_quantity@7 as cs_quantity, cs_ext_sales_price@8 as cs_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, cs_sold_date_sk@5, cs_item_sk@6, cs_order_number@7, cs_quantity@8, cs_ext_sales_price@9] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet + │ ProjectionExec: expr=[d_year@6 as d_year, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, ss_quantity@0 - coalesce(sr_return_quantity@7, 0) as sales_cnt, ss_ext_sales_price@1 - coalesce(CAST(sr_return_amt@8 AS Decimal128(30, 15)), Some(0),30,15) as sales_amt] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(ss_ticket_number@1, sr_ticket_number@1), (ss_item_sk@0, sr_item_sk@0)], projection=[ss_quantity@2, ss_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, sr_return_quantity@11, sr_return_amt@12] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_ticket_number@2 as ss_ticket_number, ss_quantity@3 as ss_quantity, ss_ext_sales_price@4 as ss_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_ticket_number@5, ss_quantity@6, ss_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@4 as ss_sold_date_sk, ss_item_sk@5 as ss_item_sk, ss_ticket_number@6 as ss_ticket_number, ss_quantity@7 as ss_quantity, ss_ext_sales_price@8 as ss_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, ss_sold_date_sk@5, ss_item_sk@6, ss_ticket_number@7, ss_quantity@8, ss_ext_sales_price@9] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet + │ ProjectionExec: expr=[d_year@6 as d_year, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, ws_quantity@0 - coalesce(wr_return_quantity@7, 0) as sales_cnt, ws_ext_sales_price@1 - coalesce(CAST(wr_return_amt@8 AS Decimal128(30, 15)), Some(0),30,15) as sales_amt] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(ws_order_number@1, wr_order_number@1), (ws_item_sk@0, wr_item_sk@0)], projection=[ws_quantity@2, ws_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, wr_return_quantity@11, wr_return_amt@12] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ws_item_sk@1 as ws_item_sk, ws_order_number@2 as ws_order_number, ws_quantity@3 as ws_quantity, ws_ext_sales_price@4 as ws_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@0)], projection=[d_year@1, ws_item_sk@4, ws_order_number@5, ws_quantity@6, ws_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@4 as ws_sold_date_sk, ws_item_sk@5 as ws_item_sk, ws_order_number@6 as ws_order_number, ws_quantity@7 as ws_quantity, ws_ext_sales_price@8 as ws_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, ws_sold_date_sk@5, ws_item_sk@6, ws_order_number@7, ws_quantity@8, ws_ext_sales_price@9] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category], file_type=parquet + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@4 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 6), input_partitions=3 - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: ss_store_sk@2 IS NULL, projection=[ss_sold_date_sk@0, ss_item_sk@1, ss_ext_sales_price@3] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=ss_store_sk@2 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=ss_store_sk_null_count@0 > 0, required_guarantees=[] + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 6), input_partitions=3 - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_sold_date_sk@0], 6), input_partitions=3 - │ ProjectionExec: expr=[ws_sold_date_sk@1 as ws_sold_date_sk, ws_ext_sales_price@2 as ws_ext_sales_price, i_category@0 as i_category] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_category@1, ws_sold_date_sk@2, ws_ext_sales_price@4] - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category], file_type=parquet + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@4 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 6), input_partitions=3 - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: ws_ship_customer_sk@2 IS NULL, projection=[ws_sold_date_sk@0, ws_item_sk@1, ws_ext_sales_price@3] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ship_customer_sk, ws_ext_sales_price], file_type=parquet, predicate=ws_ship_customer_sk@2 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=ws_ship_customer_sk_null_count@0 > 0, required_guarantees=[] + │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@4 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] └────────────────────────────────────────────────── - ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 6), input_partitions=3 - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 12 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id], aggr=[sum(sales_detail.sales_cnt), sum(sales_detail.sales_amt)], ordering_mode=PartiallySorted([0]) + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sales_cnt@5 as sales_cnt, sales_amt@6 as sales_amt], aggr=[], ordering_mode=PartiallySorted([0]) + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_sold_date_sk@0], 6), input_partitions=3 - │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_ext_sales_price@2 as cs_ext_sales_price, i_category@0 as i_category] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_category@1, cs_sold_date_sk@2, cs_ext_sales_price@4] - │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ RepartitionExec: partitioning=Hash([d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, sales_cnt@5, sales_amt@6], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sales_cnt@5 as sales_cnt, sales_amt@6 as sales_amt], aggr=[], ordering_mode=PartiallySorted([0]) + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ ProjectionExec: expr=[d_year@6 as d_year, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, cs_quantity@0 - coalesce(cr_return_quantity@7, 0) as sales_cnt, cs_ext_sales_price@1 - coalesce(CAST(cr_return_amount@8 AS Decimal128(30, 15)), Some(0),30,15) as sales_amt] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(cs_order_number@1, cr_order_number@1), (cs_item_sk@0, cr_item_sk@0)], projection=[cs_quantity@2, cs_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, cr_return_quantity@11, cr_return_amount@12] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[cs_item_sk@1 as cs_item_sk, cs_order_number@2 as cs_order_number, cs_quantity@3 as cs_quantity, cs_ext_sales_price@4 as cs_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@0)], projection=[d_year@1, cs_item_sk@4, cs_order_number@5, cs_quantity@6, cs_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@4 as cs_sold_date_sk, cs_item_sk@5 as cs_item_sk, cs_order_number@6 as cs_order_number, cs_quantity@7 as cs_quantity, cs_ext_sales_price@8 as cs_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, cs_sold_date_sk@5, cs_item_sk@6, cs_order_number@7, cs_quantity@8, cs_ext_sales_price@9] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet + │ ProjectionExec: expr=[d_year@6 as d_year, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, ss_quantity@0 - coalesce(sr_return_quantity@7, 0) as sales_cnt, ss_ext_sales_price@1 - coalesce(CAST(sr_return_amt@8 AS Decimal128(30, 15)), Some(0),30,15) as sales_amt] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(ss_ticket_number@1, sr_ticket_number@1), (ss_item_sk@0, sr_item_sk@0)], projection=[ss_quantity@2, ss_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, sr_return_quantity@11, sr_return_amt@12] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_ticket_number@2 as ss_ticket_number, ss_quantity@3 as ss_quantity, ss_ext_sales_price@4 as ss_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_ticket_number@5, ss_quantity@6, ss_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@4 as ss_sold_date_sk, ss_item_sk@5 as ss_item_sk, ss_ticket_number@6 as ss_ticket_number, ss_quantity@7 as ss_quantity, ss_ext_sales_price@8 as ss_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, ss_sold_date_sk@5, ss_item_sk@6, ss_ticket_number@7, ss_quantity@8, ss_ext_sales_price@9] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet + │ ProjectionExec: expr=[d_year@6 as d_year, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, ws_quantity@0 - coalesce(wr_return_quantity@7, 0) as sales_cnt, ws_ext_sales_price@1 - coalesce(CAST(wr_return_amt@8 AS Decimal128(30, 15)), Some(0),30,15) as sales_amt] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(ws_order_number@1, wr_order_number@1), (ws_item_sk@0, wr_item_sk@0)], projection=[ws_quantity@2, ws_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, wr_return_quantity@11, wr_return_amt@12] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ws_item_sk@1 as ws_item_sk, ws_order_number@2 as ws_order_number, ws_quantity@3 as ws_quantity, ws_ext_sales_price@4 as ws_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@0)], projection=[d_year@1, ws_item_sk@4, ws_order_number@5, ws_quantity@6, ws_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@4 as ws_sold_date_sk, ws_item_sk@5 as ws_item_sk, ws_order_number@6 as ws_order_number, ws_quantity@7 as ws_quantity, ws_ext_sales_price@8 as ws_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, ws_sold_date_sk@5, ws_item_sk@6, ws_order_number@7, ws_quantity@8, ws_ext_sales_price@9] + │ CoalescePartitionsExec + │ [Stage 14] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category], file_type=parquet + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@4 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] └────────────────────────────────────────────────── - ┌───── Stage 11 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 6), input_partitions=3 - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: cs_ship_addr_sk@1 IS NULL, projection=[cs_sold_date_sk@0, cs_item_sk@2, cs_ext_sales_price@3] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_ship_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=cs_ship_addr_sk@1 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=cs_ship_addr_sk_null_count@0 > 0, required_guarantees=[] + │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@4 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@4 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] └────────────────────────────────────────────────── "); Ok(()) } #[tokio::test] - async fn test_tpcds_77() -> Result<()> { - let display = test_tpcds_query(77).await?; + async fn test_tpcds_76() -> Result<()> { + let display = test_tpcds_query(76).await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [channel@0 ASC, id@1 ASC, returns_@3 DESC], fetch=100 - │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, id@1 ASC, returns_@3 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[channel@0 as channel, id@1 as id, sum(x.sales)@3 as sales, sum(x.returns_)@4 as returns_, sum(x.profit)@5 as profit] - │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, id@1 as id, __grouping_id@2 as __grouping_id], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([channel@0, id@1, __grouping_id@2], 3), input_partitions=9 - │ AggregateExec: mode=Partial, gby=[(NULL as channel, NULL as id), (channel@0 as channel, NULL as id), (channel@0 as channel, id@1 as id)], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] - │ UnionExec - │ ProjectionExec: expr=[store channel as channel, CAST(s_store_sk@0 AS Float64) as id, sales@1 as sales, coalesce(CAST(returns_@4 AS Decimal128(22, 2)), Some(0),22,2) as returns_, profit@2 - coalesce(CAST(profit_loss@5 AS Decimal128(22, 2)), Some(0),22,2) as profit] - │ ProjectionExec: expr=[s_store_sk@3 as s_store_sk, sales@4 as sales, profit@5 as profit, s_store_sk@0 as s_store_sk, returns_@1 as returns_, profit_loss@2 as profit_loss] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(s_store_sk@0, s_store_sk@0)] - │ CoalescePartitionsExec - │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, sum(store_returns.sr_return_amt)@1 as returns_, sum(store_returns.sr_net_loss)@2 as profit_loss] - │ AggregateExec: mode=FinalPartitioned, gby=[s_store_sk@0 as s_store_sk], aggr=[sum(store_returns.sr_return_amt), sum(store_returns.sr_net_loss)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([s_store_sk@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[s_store_sk@2 as s_store_sk], aggr=[sum(store_returns.sr_return_amt), sum(store_returns.sr_net_loss)] - │ ProjectionExec: expr=[sr_return_amt@1 as sr_return_amt, sr_net_loss@2 as sr_net_loss, s_store_sk@0 as s_store_sk] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, sr_store_sk@0)], projection=[s_store_sk@0, sr_return_amt@3, sr_net_loss@4] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, sr_returned_date_sk@0)], projection=[sr_store_sk@3, sr_return_amt@4, sr_net_loss@5] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_store_sk, sr_return_amt, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, sum(store_sales.ss_ext_sales_price)@1 as sales, sum(store_sales.ss_net_profit)@2 as profit] - │ AggregateExec: mode=FinalPartitioned, gby=[s_store_sk@0 as s_store_sk], aggr=[sum(store_sales.ss_ext_sales_price), sum(store_sales.ss_net_profit)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([s_store_sk@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[s_store_sk@2 as s_store_sk], aggr=[sum(store_sales.ss_ext_sales_price), sum(store_sales.ss_net_profit)] - │ ProjectionExec: expr=[ss_ext_sales_price@1 as ss_ext_sales_price, ss_net_profit@2 as ss_net_profit, s_store_sk@0 as s_store_sk] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)], projection=[s_store_sk@0, ss_ext_sales_price@3, ss_net_profit@4] - │ CoalescePartitionsExec - │ [Stage 3] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_store_sk@3, ss_ext_sales_price@4, ss_net_profit@5] - │ CoalescePartitionsExec - │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ ProjectionExec: expr=[catalog channel as channel, cs_call_center_sk@0 as id, sales@1 as sales, CAST(returns_@3 AS Decimal128(22, 2)) as returns_, CAST(profit@2 - profit_loss@4 AS Decimal128(23, 2)) as profit] - │ CrossJoinExec - │ CoalescePartitionsExec - │ ProjectionExec: expr=[cs_call_center_sk@0 as cs_call_center_sk, sum(catalog_sales.cs_ext_sales_price)@1 as sales, sum(catalog_sales.cs_net_profit)@2 as profit] - │ AggregateExec: mode=FinalPartitioned, gby=[cs_call_center_sk@0 as cs_call_center_sk], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(catalog_sales.cs_net_profit)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_call_center_sk@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[cs_call_center_sk@0 as cs_call_center_sk], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(catalog_sales.cs_net_profit)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_call_center_sk@3, cs_ext_sales_price@4, cs_net_profit@5] - │ CoalescePartitionsExec - │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_ext_sales_price, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] - │ ProjectionExec: expr=[sum(catalog_returns.cr_return_amount)@1 as returns_, sum(catalog_returns.cr_net_loss)@2 as profit_loss] - │ AggregateExec: mode=FinalPartitioned, gby=[cr_call_center_sk@0 as cr_call_center_sk], aggr=[sum(catalog_returns.cr_return_amount), sum(catalog_returns.cr_net_loss)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cr_call_center_sk@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[cr_call_center_sk@0 as cr_call_center_sk], aggr=[sum(catalog_returns.cr_return_amount), sum(catalog_returns.cr_net_loss)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cr_returned_date_sk@0)], projection=[cr_call_center_sk@2, cr_return_amount@3, cr_net_loss@4] - │ CoalescePartitionsExec - │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_call_center_sk, cr_return_amount, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] - │ ProjectionExec: expr=[web channel as channel, CAST(wp_web_page_sk@0 AS Float64) as id, sales@1 as sales, coalesce(CAST(returns_@4 AS Decimal128(22, 2)), Some(0),22,2) as returns_, profit@2 - coalesce(CAST(profit_loss@5 AS Decimal128(22, 2)), Some(0),22,2) as profit] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(wp_web_page_sk@0, wp_web_page_sk@0)] - │ CoalescePartitionsExec - │ ProjectionExec: expr=[wp_web_page_sk@0 as wp_web_page_sk, sum(web_sales.ws_ext_sales_price)@1 as sales, sum(web_sales.ws_net_profit)@2 as profit] - │ AggregateExec: mode=FinalPartitioned, gby=[wp_web_page_sk@0 as wp_web_page_sk], aggr=[sum(web_sales.ws_ext_sales_price), sum(web_sales.ws_net_profit)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([wp_web_page_sk@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[wp_web_page_sk@2 as wp_web_page_sk], aggr=[sum(web_sales.ws_ext_sales_price), sum(web_sales.ws_net_profit)] - │ ProjectionExec: expr=[ws_ext_sales_price@1 as ws_ext_sales_price, ws_net_profit@2 as ws_net_profit, wp_web_page_sk@0 as wp_web_page_sk] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_page.wp_web_page_sk AS Float64)@1, ws_web_page_sk@0)], projection=[wp_web_page_sk@0, ws_ext_sales_price@3, ws_net_profit@4] - │ CoalescePartitionsExec - │ [Stage 7] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_web_page_sk@3, ws_ext_sales_price@4, ws_net_profit@5] - │ CoalescePartitionsExec - │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_web_page_sk, ws_ext_sales_price, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] - │ ProjectionExec: expr=[wp_web_page_sk@0 as wp_web_page_sk, sum(web_returns.wr_return_amt)@1 as returns_, sum(web_returns.wr_net_loss)@2 as profit_loss] - │ AggregateExec: mode=FinalPartitioned, gby=[wp_web_page_sk@0 as wp_web_page_sk], aggr=[sum(web_returns.wr_return_amt), sum(web_returns.wr_net_loss)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([wp_web_page_sk@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[wp_web_page_sk@2 as wp_web_page_sk], aggr=[sum(web_returns.wr_return_amt), sum(web_returns.wr_net_loss)] - │ ProjectionExec: expr=[wr_return_amt@1 as wr_return_amt, wr_net_loss@2 as wr_net_loss, wp_web_page_sk@0 as wp_web_page_sk] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_page.wp_web_page_sk AS Float64)@1, wr_web_page_sk@0)], projection=[wp_web_page_sk@0, wr_return_amt@3, wr_net_loss@4] - │ CoalescePartitionsExec - │ [Stage 9] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, wr_returned_date_sk@0)], projection=[wr_web_page_sk@3, wr_return_amt@4, wr_net_loss@5] - │ CoalescePartitionsExec - │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_returned_date_sk, wr_web_page_sk, wr_return_amt, wr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ SortPreservingMergeExec: [channel@0 ASC, col_name@1 ASC, d_year@2 ASC, d_qoy@3 ASC, i_category@4 ASC], fetch=100 + │ [Stage 14] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] - │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p1] t1:[p2..p3] - │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p1] t1:[p2..p3] - │ ProjectionExec: expr=[wp_web_page_sk@0 as wp_web_page_sk, CAST(wp_web_page_sk@0 AS Float64) as CAST(web_page.wp_web_page_sk AS Float64)] - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-3.parquet]]}, projection=[wp_web_page_sk], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 9 ── Tasks: t0:[p0..p1] t1:[p2..p3] - │ ProjectionExec: expr=[wp_web_page_sk@0 as wp_web_page_sk, CAST(wp_web_page_sk@0 AS Float64) as CAST(web_page.wp_web_page_sk AS Float64)] - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-3.parquet]]}, projection=[wp_web_page_sk], file_type=parquet + ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, col_name@1 ASC, d_year@2 ASC, d_qoy@3 ASC, i_category@4 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[channel@0 as channel, col_name@1 as col_name, d_year@2 as d_year, d_qoy@3 as d_qoy, i_category@4 as i_category, count(Int64(1))@5 as sales_cnt, sum(foo.ext_sales_price)@6 as sales_amt] + │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, col_name@1 as col_name, d_year@2 as d_year, d_qoy@3 as d_qoy, i_category@4 as i_category], aggr=[count(Int64(1)), sum(foo.ext_sales_price)] + │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=4 └────────────────────────────────────────────────── - ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + ┌───── Stage 13 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([channel@0, col_name@1, d_year@2, d_qoy@3, i_category@4], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[channel@0 as channel, col_name@1 as col_name, d_year@2 as d_year, d_qoy@3 as d_qoy, i_category@4 as i_category], aggr=[count(Int64(1)), sum(foo.ext_sales_price)], ordering_mode=PartiallySorted([0, 1]) + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2(0/2)] t3:[c2(1/2)] + │ ProjectionExec: expr=[store as channel, ss_store_sk as col_name, d_year@4 as d_year, d_qoy@5 as d_qoy, i_category@2 as i_category, ss_ext_sales_price@1 as ext_sales_price] + │ ProjectionExec: expr=[ss_sold_date_sk@3 as ss_sold_date_sk, ss_ext_sales_price@4 as ss_ext_sales_price, i_category@5 as i_category, d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@0)], projection=[d_date_sk@0, d_year@1, d_qoy@2, ss_sold_date_sk@4, ss_ext_sales_price@5, i_category@6] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[web as channel, ws_ship_customer_sk as col_name, d_year@4 as d_year, d_qoy@5 as d_qoy, i_category@2 as i_category, ws_ext_sales_price@1 as ext_sales_price] + │ ProjectionExec: expr=[ws_sold_date_sk@3 as ws_sold_date_sk, ws_ext_sales_price@4 as ws_ext_sales_price, i_category@5 as i_category, d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ws_sold_date_sk@0)], projection=[d_date_sk@0, d_year@1, d_qoy@2, ws_sold_date_sk@4, ws_ext_sales_price@5, i_category@6] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[catalog as channel, cs_ship_addr_sk as col_name, d_year@4 as d_year, d_qoy@5 as d_qoy, i_category@2 as i_category, cs_ext_sales_price@1 as ext_sales_price] + │ ProjectionExec: expr=[cs_sold_date_sk@3 as cs_sold_date_sk, cs_ext_sales_price@4 as cs_ext_sales_price, i_category@5 as i_category, d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, cs_sold_date_sk@0)], projection=[d_date_sk@0, d_year@1, d_qoy@2, cs_sold_date_sk@4, cs_ext_sales_price@5, i_category@6] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 3), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_sold_date_sk@0], 3), input_partitions=3 + │ ProjectionExec: expr=[ss_sold_date_sk@1 as ss_sold_date_sk, ss_ext_sales_price@2 as ss_ext_sales_price, i_category@0 as i_category] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_category@1, ss_sold_date_sk@2, ss_ext_sales_price@4] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 6), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ss_store_sk@2 IS NULL, projection=[ss_sold_date_sk@0, ss_item_sk@1, ss_ext_sales_price@3] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=ss_store_sk@2 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=ss_store_sk_null_count@0 > 0, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 3), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_sold_date_sk@0], 3), input_partitions=3 + │ ProjectionExec: expr=[ws_sold_date_sk@1 as ws_sold_date_sk, ws_ext_sales_price@2 as ws_ext_sales_price, i_category@0 as i_category] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_category@1, ws_sold_date_sk@2, ws_ext_sales_price@4] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 6), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: ws_ship_customer_sk@2 IS NULL, projection=[ws_sold_date_sk@0, ws_item_sk@1, ws_ext_sales_price@3] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ship_customer_sk, ws_ext_sales_price], file_type=parquet, predicate=ws_ship_customer_sk@2 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=ws_ship_customer_sk_null_count@0 > 0, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 6), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_sold_date_sk@0], 6), input_partitions=3 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_ext_sales_price@2 as cs_ext_sales_price, i_category@0 as i_category] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_category@1, cs_sold_date_sk@2, cs_ext_sales_price@4] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 6), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: cs_ship_addr_sk@1 IS NULL, projection=[cs_sold_date_sk@0, cs_item_sk@2, cs_ext_sales_price@3] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_ship_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=cs_ship_addr_sk@1 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=cs_ship_addr_sk_null_count@0 > 0, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_77() -> Result<()> { + let display = test_tpcds_query(77).await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [channel@0 ASC, id@1 ASC, returns_@3 DESC], fetch=100 + │ [Stage 12] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, id@1 ASC, returns_@3 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[channel@0 as channel, id@1 as id, sum(x.sales)@3 as sales, sum(x.returns_)@4 as returns_, sum(x.profit)@5 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, id@1 as id, __grouping_id@2 as __grouping_id], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([channel@0, id@1, __grouping_id@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[(NULL as channel, NULL as id), (channel@0 as channel, NULL as id), (channel@0 as channel, id@1 as id)], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ ProjectionExec: expr=[store channel as channel, CAST(s_store_sk@0 AS Float64) as id, sales@1 as sales, coalesce(CAST(returns_@4 AS Decimal128(22, 2)), Some(0),22,2) as returns_, profit@2 - coalesce(CAST(profit_loss@5 AS Decimal128(22, 2)), Some(0),22,2) as profit] + │ ProjectionExec: expr=[s_store_sk@3 as s_store_sk, sales@4 as sales, profit@5 as profit, s_store_sk@0 as s_store_sk, returns_@1 as returns_, profit_loss@2 as profit_loss] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(s_store_sk@0, s_store_sk@0)] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, sum(store_returns.sr_return_amt)@1 as returns_, sum(store_returns.sr_net_loss)@2 as profit_loss] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_sk@0 as s_store_sk], aggr=[sum(store_returns.sr_return_amt), sum(store_returns.sr_net_loss)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([s_store_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[s_store_sk@2 as s_store_sk], aggr=[sum(store_returns.sr_return_amt), sum(store_returns.sr_net_loss)] + │ ProjectionExec: expr=[sr_return_amt@1 as sr_return_amt, sr_net_loss@2 as sr_net_loss, s_store_sk@0 as s_store_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, sr_store_sk@0)], projection=[s_store_sk@0, sr_return_amt@3, sr_net_loss@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, sr_returned_date_sk@0)], projection=[sr_store_sk@3, sr_return_amt@4, sr_net_loss@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_returned_date_sk, sr_store_sk, sr_return_amt, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, sum(store_sales.ss_ext_sales_price)@1 as sales, sum(store_sales.ss_net_profit)@2 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_sk@0 as s_store_sk], aggr=[sum(store_sales.ss_ext_sales_price), sum(store_sales.ss_net_profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([s_store_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[s_store_sk@2 as s_store_sk], aggr=[sum(store_sales.ss_ext_sales_price), sum(store_sales.ss_net_profit)] + │ ProjectionExec: expr=[ss_ext_sales_price@1 as ss_ext_sales_price, ss_net_profit@2 as ss_net_profit, s_store_sk@0 as s_store_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)], projection=[s_store_sk@0, ss_ext_sales_price@3, ss_net_profit@4] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_store_sk@3, ss_ext_sales_price@4, ss_net_profit@5] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[catalog channel as channel, cs_call_center_sk@0 as id, sales@1 as sales, CAST(returns_@3 AS Decimal128(22, 2)) as returns_, CAST(profit@2 - profit_loss@4 AS Decimal128(23, 2)) as profit] + │ CrossJoinExec + │ CoalescePartitionsExec + │ ProjectionExec: expr=[cs_call_center_sk@0 as cs_call_center_sk, sum(catalog_sales.cs_ext_sales_price)@1 as sales, sum(catalog_sales.cs_net_profit)@2 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[cs_call_center_sk@0 as cs_call_center_sk], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(catalog_sales.cs_net_profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_call_center_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cs_call_center_sk@0 as cs_call_center_sk], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(catalog_sales.cs_net_profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_call_center_sk@3, cs_ext_sales_price@4, cs_net_profit@5] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_ext_sales_price, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[sum(catalog_returns.cr_return_amount)@1 as returns_, sum(catalog_returns.cr_net_loss)@2 as profit_loss] + │ AggregateExec: mode=FinalPartitioned, gby=[cr_call_center_sk@0 as cr_call_center_sk], aggr=[sum(catalog_returns.cr_return_amount), sum(catalog_returns.cr_net_loss)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cr_call_center_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cr_call_center_sk@0 as cr_call_center_sk], aggr=[sum(catalog_returns.cr_return_amount), sum(catalog_returns.cr_net_loss)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cr_returned_date_sk@0)], projection=[cr_call_center_sk@2, cr_return_amount@3, cr_net_loss@4] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_returned_date_sk, cr_call_center_sk, cr_return_amount, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[web channel as channel, CAST(wp_web_page_sk@0 AS Float64) as id, sales@1 as sales, coalesce(CAST(returns_@4 AS Decimal128(22, 2)), Some(0),22,2) as returns_, profit@2 - coalesce(CAST(profit_loss@5 AS Decimal128(22, 2)), Some(0),22,2) as profit] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(wp_web_page_sk@0, wp_web_page_sk@0)] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[wp_web_page_sk@0 as wp_web_page_sk, sum(web_sales.ws_ext_sales_price)@1 as sales, sum(web_sales.ws_net_profit)@2 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[wp_web_page_sk@0 as wp_web_page_sk], aggr=[sum(web_sales.ws_ext_sales_price), sum(web_sales.ws_net_profit)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([wp_web_page_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[wp_web_page_sk@2 as wp_web_page_sk], aggr=[sum(web_sales.ws_ext_sales_price), sum(web_sales.ws_net_profit)] + │ ProjectionExec: expr=[ws_ext_sales_price@1 as ws_ext_sales_price, ws_net_profit@2 as ws_net_profit, wp_web_page_sk@0 as wp_web_page_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_page.wp_web_page_sk AS Float64)@1, ws_web_page_sk@0)], projection=[wp_web_page_sk@0, ws_ext_sales_price@3, ws_net_profit@4] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_web_page_sk@3, ws_ext_sales_price@4, ws_net_profit@5] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_web_page_sk, ws_ext_sales_price, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[wp_web_page_sk@0 as wp_web_page_sk, sum(web_returns.wr_return_amt)@1 as returns_, sum(web_returns.wr_net_loss)@2 as profit_loss] + │ AggregateExec: mode=FinalPartitioned, gby=[wp_web_page_sk@0 as wp_web_page_sk], aggr=[sum(web_returns.wr_return_amt), sum(web_returns.wr_net_loss)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([wp_web_page_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[wp_web_page_sk@2 as wp_web_page_sk], aggr=[sum(web_returns.wr_return_amt), sum(web_returns.wr_net_loss)] + │ ProjectionExec: expr=[wr_return_amt@1 as wr_return_amt, wr_net_loss@2 as wr_net_loss, wp_web_page_sk@0 as wp_web_page_sk] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_page.wp_web_page_sk AS Float64)@1, wr_web_page_sk@0)], projection=[wp_web_page_sk@0, wr_return_amt@3, wr_net_loss@4] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, wr_returned_date_sk@0)], projection=[wr_web_page_sk@3, wr_return_amt@4, wr_net_loss@5] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_returned_date_sk, wr_web_page_sk, wr_return_amt, wr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[wp_web_page_sk@0 as wp_web_page_sk, CAST(wp_web_page_sk@0 AS Float64) as CAST(web_page.wp_web_page_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-3.parquet]]}, projection=[wp_web_page_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[wp_web_page_sk@0 as wp_web_page_sk, CAST(wp_web_page_sk@0 AS Float64) as CAST(web_page.wp_web_page_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-3.parquet]]}, projection=[wp_web_page_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── "); Ok(()) } @@ -8407,211 +8455,214 @@ mod tests { │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, id@1 ASC], preserve_partitioning=[true] │ ProjectionExec: expr=[channel@0 as channel, id@1 as id, sum(x.sales)@3 as sales, sum(x.returns_)@4 as returns_, sum(x.profit)@5 as profit] │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, id@1 as id, __grouping_id@2 as __grouping_id], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([channel@0, id@1, __grouping_id@2], 3), input_partitions=9 - │ AggregateExec: mode=Partial, gby=[(NULL as channel, NULL as id), (channel@0 as channel, NULL as id), (channel@0 as channel, id@1 as id)], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] - │ UnionExec - │ ProjectionExec: expr=[store channel as channel, concat(store, store_id@0) as id, sales@1 as sales, returns_@2 as returns_, profit@3 as profit] - │ ProjectionExec: expr=[s_store_id@0 as store_id, sum(store_sales.ss_ext_sales_price)@1 as sales, sum(coalesce(store_returns.sr_return_amt,Int64(0)))@2 as returns_, sum(store_sales.ss_net_profit - coalesce(store_returns.sr_net_loss,Int64(0)))@3 as profit] - │ AggregateExec: mode=FinalPartitioned, gby=[s_store_id@0 as s_store_id], aggr=[sum(store_sales.ss_ext_sales_price), sum(coalesce(store_returns.sr_return_amt,Int64(0))), sum(store_sales.ss_net_profit - coalesce(store_returns.sr_net_loss,Int64(0)))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([s_store_id@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[s_store_id@4 as s_store_id], aggr=[sum(store_sales.ss_ext_sales_price), sum(coalesce(store_returns.sr_return_amt,Int64(0))), sum(store_sales.ss_net_profit - coalesce(store_returns.sr_net_loss,Int64(0)))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(promotion.p_promo_sk AS Float64)@1, ss_promo_sk@0)], projection=[ss_ext_sales_price@3, ss_net_profit@4, sr_return_amt@5, sr_net_loss@6, s_store_id@7] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[ss_promo_sk@2, ss_ext_sales_price@3, ss_net_profit@4, sr_return_amt@5, sr_net_loss@6, s_store_id@7] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_promo_sk@2 as ss_promo_sk, ss_ext_sales_price@3 as ss_ext_sales_price, ss_net_profit@4 as ss_net_profit, sr_return_amt@5 as sr_return_amt, sr_net_loss@6 as sr_net_loss, s_store_id@0 as s_store_id] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, ss_store_sk@1)], projection=[s_store_id@1, ss_item_sk@3, ss_promo_sk@5, ss_ext_sales_price@6, ss_net_profit@7, sr_return_amt@8, sr_net_loss@9] - │ CoalescePartitionsExec - │ [Stage 3] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_promo_sk@5, ss_ext_sales_price@6, ss_net_profit@7, sr_return_amt@8, sr_net_loss@9] - │ CoalescePartitionsExec - │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ ProjectionExec: expr=[ss_sold_date_sk@2 as ss_sold_date_sk, ss_item_sk@3 as ss_item_sk, ss_store_sk@4 as ss_store_sk, ss_promo_sk@5 as ss_promo_sk, ss_ext_sales_price@6 as ss_ext_sales_price, ss_net_profit@7 as ss_net_profit, sr_return_amt@0 as sr_return_amt, sr_net_loss@1 as sr_net_loss] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(sr_item_sk@0, ss_item_sk@1), (sr_ticket_number@1, ss_ticket_number@4)], projection=[sr_return_amt@2, sr_net_loss@3, ss_sold_date_sk@4, ss_item_sk@5, ss_store_sk@6, ss_promo_sk@7, ss_ext_sales_price@9, ss_net_profit@10] - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ ProjectionExec: expr=[catalog channel as channel, concat(catalog_page, catalog_page_id@0) as id, sales@1 as sales, returns_@2 as returns_, profit@3 as profit] - │ ProjectionExec: expr=[cp_catalog_page_id@0 as catalog_page_id, sum(catalog_sales.cs_ext_sales_price)@1 as sales, sum(coalesce(catalog_returns.cr_return_amount,Int64(0)))@2 as returns_, sum(catalog_sales.cs_net_profit - coalesce(catalog_returns.cr_net_loss,Int64(0)))@3 as profit] - │ AggregateExec: mode=FinalPartitioned, gby=[cp_catalog_page_id@0 as cp_catalog_page_id], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(coalesce(catalog_returns.cr_return_amount,Int64(0))), sum(catalog_sales.cs_net_profit - coalesce(catalog_returns.cr_net_loss,Int64(0)))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cp_catalog_page_id@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[cp_catalog_page_id@4 as cp_catalog_page_id], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(coalesce(catalog_returns.cr_return_amount,Int64(0))), sum(catalog_sales.cs_net_profit - coalesce(catalog_returns.cr_net_loss,Int64(0)))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(promotion.p_promo_sk AS Float64)@1, cs_promo_sk@0)], projection=[cs_ext_sales_price@3, cs_net_profit@4, cr_return_amount@5, cr_net_loss@6, cp_catalog_page_id@7] - │ CoalescePartitionsExec - │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@0)], projection=[cs_promo_sk@2, cs_ext_sales_price@3, cs_net_profit@4, cr_return_amount@5, cr_net_loss@6, cp_catalog_page_id@7] - │ CoalescePartitionsExec - │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cs_catalog_page_sk@0, CAST(catalog_page.cp_catalog_page_sk AS Float64)@2)], projection=[cs_item_sk@1, cs_promo_sk@2, cs_ext_sales_price@3, cs_net_profit@4, cr_return_amount@5, cr_net_loss@6, cp_catalog_page_id@8] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_catalog_page_sk@0], 3), input_partitions=3 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_catalog_page_sk@3, cs_item_sk@4, cs_promo_sk@5, cs_ext_sales_price@6, cs_net_profit@7, cr_return_amount@8, cr_net_loss@9] - │ CoalescePartitionsExec - │ [Stage 9] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ ProjectionExec: expr=[cs_sold_date_sk@2 as cs_sold_date_sk, cs_catalog_page_sk@3 as cs_catalog_page_sk, cs_item_sk@4 as cs_item_sk, cs_promo_sk@5 as cs_promo_sk, cs_ext_sales_price@6 as cs_ext_sales_price, cs_net_profit@7 as cs_net_profit, cr_return_amount@0 as cr_return_amount, cr_net_loss@1 as cr_net_loss] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(cr_item_sk@0, cs_item_sk@2), (cr_order_number@1, cs_order_number@4)], projection=[cr_return_amount@2, cr_net_loss@3, cs_sold_date_sk@4, cs_catalog_page_sk@5, cs_item_sk@6, cs_promo_sk@7, cs_ext_sales_price@9, cs_net_profit@10] - │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ ProjectionExec: expr=[web channel as channel, concat(web_site, web_site_id@0) as id, sales@1 as sales, returns_@2 as returns_, profit@3 as profit] - │ ProjectionExec: expr=[web_site_id@0 as web_site_id, sum(web_sales.ws_ext_sales_price)@1 as sales, sum(coalesce(web_returns.wr_return_amt,Int64(0)))@2 as returns_, sum(web_sales.ws_net_profit - coalesce(web_returns.wr_net_loss,Int64(0)))@3 as profit] - │ AggregateExec: mode=FinalPartitioned, gby=[web_site_id@0 as web_site_id], aggr=[sum(web_sales.ws_ext_sales_price), sum(coalesce(web_returns.wr_return_amt,Int64(0))), sum(web_sales.ws_net_profit - coalesce(web_returns.wr_net_loss,Int64(0)))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([web_site_id@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[web_site_id@4 as web_site_id], aggr=[sum(web_sales.ws_ext_sales_price), sum(coalesce(web_returns.wr_return_amt,Int64(0))), sum(web_sales.ws_net_profit - coalesce(web_returns.wr_net_loss,Int64(0)))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_promo_sk@0, CAST(promotion.p_promo_sk AS Float64)@1)], projection=[ws_ext_sales_price@1, ws_net_profit@2, wr_return_amt@3, wr_net_loss@4, web_site_id@5] - │ CoalescePartitionsExec - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@0, i_item_sk@0)], projection=[ws_promo_sk@1, ws_ext_sales_price@2, ws_net_profit@3, wr_return_amt@4, wr_net_loss@5, web_site_id@6] - │ CoalescePartitionsExec - │ ProjectionExec: expr=[ws_item_sk@1 as ws_item_sk, ws_promo_sk@2 as ws_promo_sk, ws_ext_sales_price@3 as ws_ext_sales_price, ws_net_profit@4 as ws_net_profit, wr_return_amt@5 as wr_return_amt, wr_net_loss@6 as wr_net_loss, web_site_id@0 as web_site_id] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_site.web_site_sk AS Float64)@2, ws_web_site_sk@1)], projection=[web_site_id@1, ws_item_sk@3, ws_promo_sk@5, ws_ext_sales_price@6, ws_net_profit@7, wr_return_amt@8, wr_net_loss@9] - │ CoalescePartitionsExec - │ [Stage 13] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_web_site_sk@4, ws_promo_sk@5, ws_ext_sales_price@6, ws_net_profit@7, wr_return_amt@8, wr_net_loss@9] - │ CoalescePartitionsExec - │ [Stage 14] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 - │ ProjectionExec: expr=[ws_sold_date_sk@2 as ws_sold_date_sk, ws_item_sk@3 as ws_item_sk, ws_web_site_sk@4 as ws_web_site_sk, ws_promo_sk@5 as ws_promo_sk, ws_ext_sales_price@6 as ws_ext_sales_price, ws_net_profit@7 as ws_net_profit, wr_return_amt@0 as wr_return_amt, wr_net_loss@1 as wr_net_loss] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(wr_item_sk@0, ws_item_sk@1), (wr_order_number@1, ws_order_number@4)], projection=[wr_return_amt@2, wr_net_loss@3, ws_sold_date_sk@4, ws_item_sk@5, ws_web_site_sk@6, ws_promo_sk@7, ws_ext_sales_price@9, ws_net_profit@10] - │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: i_current_price@1 > Some(5000),4,2, projection=[i_item_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@1 > Some(5000),4,2 AND DynamicFilter [ empty ], pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),4,2, required_guarantees=[] - │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: p_channel_tv@1 = N, projection=[p_promo_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, p_channel_tv], file_type=parquet, predicate=p_channel_tv@1 = N, pruning_predicate=p_channel_tv_null_count@2 != row_count@3 AND p_channel_tv_min@0 <= N AND N <= p_channel_tv_max@1, required_guarantees=[p_channel_tv in (N)] + │ [Stage 17] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: p_channel_tv@1 = N, projection=[p_promo_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, p_channel_tv], file_type=parquet, predicate=p_channel_tv@1 = N, pruning_predicate=p_channel_tv_null_count@2 != row_count@3 AND p_channel_tv_min@0 <= N AND N <= p_channel_tv_max@1, required_guarantees=[p_channel_tv in (N)] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: i_current_price@1 > Some(5000),4,2, projection=[i_item_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@1 > Some(5000),4,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),4,2, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p1] t1:[p2..p3] - │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_id@1 as s_store_id, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number, sr_return_amt, sr_net_loss], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_item_sk@1, ss_ticket_number@4], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_ext_sales_price, ss_net_profit], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: p_channel_tv@1 = N, projection=[p_promo_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, p_channel_tv], file_type=parquet, predicate=p_channel_tv@1 = N, pruning_predicate=p_channel_tv_null_count@2 != row_count@3 AND p_channel_tv_min@0 <= N AND N <= p_channel_tv_max@1, required_guarantees=[p_channel_tv in (N)] - └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: i_current_price@1 > Some(5000),4,2, projection=[i_item_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@1 > Some(5000),4,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),4,2, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cr_item_sk@0, cr_order_number@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_return_amount, cr_net_loss], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_item_sk@2, cs_order_number@4], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_catalog_page_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_ext_sales_price, cs_net_profit], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(catalog_page.cp_catalog_page_sk AS Float64)@2], 3), input_partitions=3 - │ ProjectionExec: expr=[cp_catalog_page_sk@0 as cp_catalog_page_sk, cp_catalog_page_id@1 as cp_catalog_page_id, CAST(cp_catalog_page_sk@0 AS Float64) as CAST(catalog_page.cp_catalog_page_sk AS Float64)] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-3.parquet]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 13 ── Tasks: t0:[p0..p1] t1:[p2..p3] - │ ProjectionExec: expr=[web_site_sk@0 as web_site_sk, web_site_id@1 as web_site_id, CAST(web_site_sk@0 AS Float64) as CAST(web_site.web_site_sk AS Float64)] - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-3.parquet]]}, projection=[web_site_sk, web_site_id], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p3..p5] - │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 15 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([wr_item_sk@0, wr_order_number@1], 3), input_partitions=3 - │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] - │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_item_sk@1, ws_order_number@4], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_site_sk, ws_promo_sk, ws_order_number, ws_ext_sales_price, ws_net_profit], file_type=parquet - └────────────────────────────────────────────────── + ┌───── Stage 17 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([channel@0, id@1, __grouping_id@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[(NULL as channel, NULL as id), (channel@0 as channel, NULL as id), (channel@0 as channel, id@1 as id)], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ ProjectionExec: expr=[store channel as channel, concat(store, store_id@0) as id, sales@1 as sales, returns_@2 as returns_, profit@3 as profit] + │ ProjectionExec: expr=[s_store_id@0 as store_id, sum(store_sales.ss_ext_sales_price)@1 as sales, sum(coalesce(store_returns.sr_return_amt,Int64(0)))@2 as returns_, sum(store_sales.ss_net_profit - coalesce(store_returns.sr_net_loss,Int64(0)))@3 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_id@0 as s_store_id], aggr=[sum(store_sales.ss_ext_sales_price), sum(coalesce(store_returns.sr_return_amt,Int64(0))), sum(store_sales.ss_net_profit - coalesce(store_returns.sr_net_loss,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([s_store_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[s_store_id@4 as s_store_id], aggr=[sum(store_sales.ss_ext_sales_price), sum(coalesce(store_returns.sr_return_amt,Int64(0))), sum(store_sales.ss_net_profit - coalesce(store_returns.sr_net_loss,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(promotion.p_promo_sk AS Float64)@1, ss_promo_sk@0)], projection=[ss_ext_sales_price@3, ss_net_profit@4, sr_return_amt@5, sr_net_loss@6, s_store_id@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[ss_promo_sk@2, ss_ext_sales_price@3, ss_net_profit@4, sr_return_amt@5, sr_net_loss@6, s_store_id@7] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_promo_sk@2 as ss_promo_sk, ss_ext_sales_price@3 as ss_ext_sales_price, ss_net_profit@4 as ss_net_profit, sr_return_amt@5 as sr_return_amt, sr_net_loss@6 as sr_net_loss, s_store_id@0 as s_store_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, ss_store_sk@1)], projection=[s_store_id@1, ss_item_sk@3, ss_promo_sk@5, ss_ext_sales_price@6, ss_net_profit@7, sr_return_amt@8, sr_net_loss@9] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_promo_sk@5, ss_ext_sales_price@6, ss_net_profit@7, sr_return_amt@8, sr_net_loss@9] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@2 as ss_sold_date_sk, ss_item_sk@3 as ss_item_sk, ss_store_sk@4 as ss_store_sk, ss_promo_sk@5 as ss_promo_sk, ss_ext_sales_price@6 as ss_ext_sales_price, ss_net_profit@7 as ss_net_profit, sr_return_amt@0 as sr_return_amt, sr_net_loss@1 as sr_net_loss] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(sr_item_sk@0, ss_item_sk@1), (sr_ticket_number@1, ss_ticket_number@4)], projection=[sr_return_amt@2, sr_net_loss@3, ss_sold_date_sk@4, ss_item_sk@5, ss_store_sk@6, ss_promo_sk@7, ss_ext_sales_price@9, ss_net_profit@10] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[catalog channel as channel, concat(catalog_page, catalog_page_id@0) as id, sales@1 as sales, returns_@2 as returns_, profit@3 as profit] + │ ProjectionExec: expr=[cp_catalog_page_id@0 as catalog_page_id, sum(catalog_sales.cs_ext_sales_price)@1 as sales, sum(coalesce(catalog_returns.cr_return_amount,Int64(0)))@2 as returns_, sum(catalog_sales.cs_net_profit - coalesce(catalog_returns.cr_net_loss,Int64(0)))@3 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[cp_catalog_page_id@0 as cp_catalog_page_id], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(coalesce(catalog_returns.cr_return_amount,Int64(0))), sum(catalog_sales.cs_net_profit - coalesce(catalog_returns.cr_net_loss,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cp_catalog_page_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cp_catalog_page_id@4 as cp_catalog_page_id], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(coalesce(catalog_returns.cr_return_amount,Int64(0))), sum(catalog_sales.cs_net_profit - coalesce(catalog_returns.cr_net_loss,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(promotion.p_promo_sk AS Float64)@1, cs_promo_sk@0)], projection=[cs_ext_sales_price@3, cs_net_profit@4, cr_return_amount@5, cr_net_loss@6, cp_catalog_page_id@7] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@0)], projection=[cs_promo_sk@2, cs_ext_sales_price@3, cs_net_profit@4, cr_return_amount@5, cr_net_loss@6, cp_catalog_page_id@7] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cs_catalog_page_sk@0, CAST(catalog_page.cp_catalog_page_sk AS Float64)@2)], projection=[cs_item_sk@1, cs_promo_sk@2, cs_ext_sales_price@3, cs_net_profit@4, cr_return_amount@5, cr_net_loss@6, cp_catalog_page_id@8] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_catalog_page_sk@0], 3), input_partitions=3 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_catalog_page_sk@3, cs_item_sk@4, cs_promo_sk@5, cs_ext_sales_price@6, cs_net_profit@7, cr_return_amount@8, cr_net_loss@9] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@2 as cs_sold_date_sk, cs_catalog_page_sk@3 as cs_catalog_page_sk, cs_item_sk@4 as cs_item_sk, cs_promo_sk@5 as cs_promo_sk, cs_ext_sales_price@6 as cs_ext_sales_price, cs_net_profit@7 as cs_net_profit, cr_return_amount@0 as cr_return_amount, cr_net_loss@1 as cr_net_loss] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(cr_item_sk@0, cs_item_sk@2), (cr_order_number@1, cs_order_number@4)], projection=[cr_return_amount@2, cr_net_loss@3, cs_sold_date_sk@4, cs_catalog_page_sk@5, cs_item_sk@6, cs_promo_sk@7, cs_ext_sales_price@9, cs_net_profit@10] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[web channel as channel, concat(web_site, web_site_id@0) as id, sales@1 as sales, returns_@2 as returns_, profit@3 as profit] + │ ProjectionExec: expr=[web_site_id@0 as web_site_id, sum(web_sales.ws_ext_sales_price)@1 as sales, sum(coalesce(web_returns.wr_return_amt,Int64(0)))@2 as returns_, sum(web_sales.ws_net_profit - coalesce(web_returns.wr_net_loss,Int64(0)))@3 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[web_site_id@0 as web_site_id], aggr=[sum(web_sales.ws_ext_sales_price), sum(coalesce(web_returns.wr_return_amt,Int64(0))), sum(web_sales.ws_net_profit - coalesce(web_returns.wr_net_loss,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([web_site_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[web_site_id@4 as web_site_id], aggr=[sum(web_sales.ws_ext_sales_price), sum(coalesce(web_returns.wr_return_amt,Int64(0))), sum(web_sales.ws_net_profit - coalesce(web_returns.wr_net_loss,Int64(0)))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_promo_sk@0, CAST(promotion.p_promo_sk AS Float64)@1)], projection=[ws_ext_sales_price@1, ws_net_profit@2, wr_return_amt@3, wr_net_loss@4, web_site_id@5] + │ CoalescePartitionsExec + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@0, i_item_sk@0)], projection=[ws_promo_sk@1, ws_ext_sales_price@2, ws_net_profit@3, wr_return_amt@4, wr_net_loss@5, web_site_id@6] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ws_item_sk@1 as ws_item_sk, ws_promo_sk@2 as ws_promo_sk, ws_ext_sales_price@3 as ws_ext_sales_price, ws_net_profit@4 as ws_net_profit, wr_return_amt@5 as wr_return_amt, wr_net_loss@6 as wr_net_loss, web_site_id@0 as web_site_id] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_site.web_site_sk AS Float64)@2, ws_web_site_sk@1)], projection=[web_site_id@1, ws_item_sk@3, ws_promo_sk@5, ws_ext_sales_price@6, ws_net_profit@7, wr_return_amt@8, wr_net_loss@9] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_web_site_sk@4, ws_promo_sk@5, ws_ext_sales_price@6, ws_net_profit@7, wr_return_amt@8, wr_net_loss@9] + │ CoalescePartitionsExec + │ [Stage 14] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@2 as ws_sold_date_sk, ws_item_sk@3 as ws_item_sk, ws_web_site_sk@4 as ws_web_site_sk, ws_promo_sk@5 as ws_promo_sk, ws_ext_sales_price@6 as ws_ext_sales_price, ws_net_profit@7 as ws_net_profit, wr_return_amt@0 as wr_return_amt, wr_net_loss@1 as wr_net_loss] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(wr_item_sk@0, ws_item_sk@1), (wr_order_number@1, ws_order_number@4)], projection=[wr_return_amt@2, wr_net_loss@3, ws_sold_date_sk@4, ws_item_sk@5, ws_web_site_sk@6, ws_promo_sk@7, ws_ext_sales_price@9, ws_net_profit@10] + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_current_price@1 > Some(5000),4,2, projection=[i_item_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@1 > Some(5000),4,2 AND DynamicFilter [ empty ], pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),4,2, required_guarantees=[] + │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: p_channel_tv@1 = N, projection=[p_promo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, p_channel_tv], file_type=parquet, predicate=p_channel_tv@1 = N, pruning_predicate=p_channel_tv_null_count@2 != row_count@3 AND p_channel_tv_min@0 <= N AND N <= p_channel_tv_max@1, required_guarantees=[p_channel_tv in (N)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: p_channel_tv@1 = N, projection=[p_promo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, p_channel_tv], file_type=parquet, predicate=p_channel_tv@1 = N, pruning_predicate=p_channel_tv_null_count@2 != row_count@3 AND p_channel_tv_min@0 <= N AND N <= p_channel_tv_max@1, required_guarantees=[p_channel_tv in (N)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_current_price@1 > Some(5000),4,2, projection=[i_item_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@1 > Some(5000),4,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),4,2, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_id@1 as s_store_id, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number, sr_return_amt, sr_net_loss], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ss_item_sk@1, ss_ticket_number@4], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_ext_sales_price, ss_net_profit], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: p_channel_tv@1 = N, projection=[p_promo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, p_channel_tv], file_type=parquet, predicate=p_channel_tv@1 = N, pruning_predicate=p_channel_tv_null_count@2 != row_count@3 AND p_channel_tv_min@0 <= N AND N <= p_channel_tv_max@1, required_guarantees=[p_channel_tv in (N)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: i_current_price@1 > Some(5000),4,2, projection=[i_item_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@1 > Some(5000),4,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),4,2, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cr_item_sk@0, cr_order_number@1], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_return_amount, cr_net_loss], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@2, cs_order_number@4], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_catalog_page_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_ext_sales_price, cs_net_profit], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CAST(catalog_page.cp_catalog_page_sk AS Float64)@2], 3), input_partitions=3 + │ ProjectionExec: expr=[cp_catalog_page_sk@0 as cp_catalog_page_sk, cp_catalog_page_id@1 as cp_catalog_page_id, CAST(cp_catalog_page_sk@0 AS Float64) as CAST(catalog_page.cp_catalog_page_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-3.parquet]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[web_site_sk@0 as web_site_sk, web_site_id@1 as web_site_id, CAST(web_site_sk@0 AS Float64) as CAST(web_site.web_site_sk AS Float64)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-3.parquet]]}, projection=[web_site_sk, web_site_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([wr_item_sk@0, wr_order_number@1], 3), input_partitions=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([ws_item_sk@1, ws_order_number@4], 3), input_partitions=3 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_site_sk, ws_promo_sk, ws_order_number, ws_ext_sales_price, ws_net_profit], file_type=parquet + └────────────────────────────────────────────────── "); Ok(()) } From a4b0b5a6265d315606e537eaa6402bbc3f5bf77f Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Mon, 29 Dec 2025 14:14:52 +0100 Subject: [PATCH 08/27] Update missing documentation (#266) * Split channel resolver in two * Simplify WorkerResolverExtension and ChannelResolverExtension * Add default builder to ArrowFlightEndpoint * Add some docs * Listen to clippy * Split get_flight_client_for_url in two * Fix conflicts * Remove unnecessary channel resolver * Improve WorkerResolver docs * Use one ChannelResolver per runtime * Improve error reporting on client connection failure * Add a from_session_builder method for constructing an InMemoryChannelResolver * Add ChannelResolver and WorkerResolver default implementations for Arcs * Update missing documentation * Add TaskEstimator docs --- docs/source/index.rst | 5 +- .../user-guide/arrow-flight-endpoint.md | 99 ++++++++++++++++++- docs/source/user-guide/channel-resolver.md | 60 ++++++++++- docs/source/user-guide/concepts.md | 14 ++- docs/source/user-guide/getting-started.md | 42 ++------ docs/source/user-guide/index.md | 3 +- docs/source/user-guide/task-estimator.md | 29 +++++- docs/source/user-guide/worker-resolver.md | 72 ++++++++++++++ 8 files changed, 277 insertions(+), 47 deletions(-) create mode 100644 docs/source/user-guide/worker-resolver.md diff --git a/docs/source/index.rst b/docs/source/index.rst index ae8165a8..1cea0739 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -15,9 +15,10 @@ how to contribute changes to the library yourself. user-guide/index user-guide/getting-started - user-guide/task-estimator - user-guide/channel-resolver + user-guide/worker-resolver user-guide/arrow-flight-endpoint + user-guide/channel-resolver + user-guide/task-estimator user-guide/concepts user-guide/how-a-distributed-plan-is-built diff --git a/docs/source/user-guide/arrow-flight-endpoint.md b/docs/source/user-guide/arrow-flight-endpoint.md index 41ebd752..1c8192f8 100644 --- a/docs/source/user-guide/arrow-flight-endpoint.md +++ b/docs/source/user-guide/arrow-flight-endpoint.md @@ -1,3 +1,98 @@ -# Building an Arrow Flight endpoint +# Building an Arrow Flight Endpoint -> WARNING: under construction +An `ArrowFlightEndpoint` is a gRPC server that implements the Arrow Flight protocol for distributed query execution. +Worker nodes run these endpoints to receive execution plans, execute them, and stream results back. + +## Overview + +The `ArrowFlightEndpoint` is the core worker component in Distributed DataFusion. It: + +- Receives serialized execution plans via Arrow Flight's `do_get` method +- Deserializes plans using protobuf and user-provided codecs +- Executes plans using the local DataFusion runtime +- Streams results back as Arrow record batches through the gRPC Arrow Flight interface + +## Creating an ArrowFlightEndpoint + +The `Default` implementation of the `ArrowFlightEndpoint` should satisfy most basic use cases + +```rust +use datafusion_distributed::{ArrowFlightEndpoint}; + +async fn main() { + let endpoint = ArrowFlightEndpoint::default(); + + Server::builder() + .add_service(endpoint.into_flight_server()) + .serve(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8000)) + .await?; + + Ok(()) +} +``` + +If you are using DataFusion though, it's very likely that you have your own custom UDFs, execution nodes, config options +etc... + +You'll need to tell the `ArrowFlightEndpoint` how to build your DataFusion sessions: + +```rust +async fn build_state(ctx: DistributedSessionBuilderContext) -> Result { + Ok(ctx + .builder + .with_scalar_functions(vec![your_custom_udf()]) + .build()) +} + +async fn main() { + let endpoint = ArrowFlightEndpoint::from_session_builder(build_sate); + + Server::builder() + .add_service(endpoint.into_flight_server()) + .serve(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8000)) + .await?; + + Ok(()) +} +``` + +### DistributedSessionBuilder + +The `DistributedSessionBuilder` is a closure or type that implements: + +```rust +#[async_trait] +pub trait DistributedSessionBuilder { + async fn build_session_state( + &self, + ctx: DistributedSessionBuilderContext, + ) -> Result; +} +``` + +It receives a `DistributedSessionBuilderContext` containing: + +- `SessionStateBuilder`: A pre-populated session state builder in which you can inject your custom stuff +- `headers`: HTTP headers from the incoming request (useful for passing metadata) + +## Serving the Endpoint + +Convert the endpoint to a gRPC service and serve it: + +```rust +use tonic::transport::Server; +use datafusion_distributed::ArrowFlightEndpoint; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + +async fn main() { + let endpoint = ArrowFlightEndpoint::default(); + + Server::builder() + .add_service(endpoint.into_flight_server()) + .serve(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080)) + .await?; +} +``` + +The `into_flight_server()` method wraps the endpoint in a `FlightServiceServer` with high message size limits ( +`usize::MAX`) to avoid chunking overhead for internal communication. diff --git a/docs/source/user-guide/channel-resolver.md b/docs/source/user-guide/channel-resolver.md index f96a92df..4375c88b 100644 --- a/docs/source/user-guide/channel-resolver.md +++ b/docs/source/user-guide/channel-resolver.md @@ -1,3 +1,61 @@ # Building a ChannelResolver -> WARNING: under construction \ No newline at end of file +This trait is optional, there's a sane default already in place that should work for most simple use cases. + +A `ChannelResolver` tells Distributed DataFusion how to build an Arrow Flight client baked by a +[tonic](https://github.com/hyperium/tonic) channel given a worker URL. + +There's a default implementation that simply connects to the given URL, builds the Arrow Flight client instance, +and caches it so that the same client instance gets reused for a query to the same URL. + +However, you might want to provide your own implementation. For that you need to take into account the following +points: + +- You will need to provide your own implementation in two places: + - in the `SessionContext` that handles your queries. + - while instantiating the `ArrowFlightEndpoint` with the `from_session_builder()` constructor. +- If you decide to build it from scratch, make sure that Arrow Flight clients are reused across + request rather than always building new ones. +- You can use this library's `DefaultChannelResolver` as a backbone for your own implementation. + If you do that, channel caching will be automatically managed for you. + +```rust +#[derive(Clone)] +struct CustomChannelResolver; + +#[async_trait] +impl ChannelResolver for CustomChannelResolver { + async fn get_flight_client_for_url(&self, url: &Url) -> Result, DataFusionError> { + // Build a custom FlightServiceClient wrapped with tower layers or something similar. + todo!() + } +} + +async fn main() { + // Make sure you build just one for the whole lifetime of your application, as it needs to be able to + // reuse Arrow Flight client instances across different queries. + let channel_resolver = CustomChannelResolver; + + let state = SessionStateBuilder::new() + // these two are mandatory. + .with_distributed_worker_resolver(todo!()) + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + // the CustomChannelResolver needs to be passed here once... + .with_distributed_channel_resolver(channel_resolver.clone()) + .build(); + + let endpoint = ArrowFlightEndpoint::from_session_builder(|ctx: DistributedSessionBuilderContext| { + let channel_resolver = channel_resolver.clone(); + async move { + // ... and here for each query that ArrowFlightEndpoint handles. + Ok(ctx.builder.with_distributed_channel_resolver(channel_resolver).build()) + } + }); + Server::builder() + .add_service(endpoint.into_flight_server()) + .serve(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8000)) + .await?; + + Ok(()) +} +``` \ No newline at end of file diff --git a/docs/source/user-guide/concepts.md b/docs/source/user-guide/concepts.md index 30df1911..d5e81007 100644 --- a/docs/source/user-guide/concepts.md +++ b/docs/source/user-guide/concepts.md @@ -41,13 +41,17 @@ you are in, you might want to return a different set of data. For example, if you are on the task with index 0 of a 3-task stage, you might want to return only the first 1/3 of the data. If you are on the task with index 2, you might want to return the last 1/3 of the data, and so on. -## [ChannelResolver](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/src/channel_resolver_ext.rs) +## [WorkerResolver](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/src/networking/worker_resolver.rs) -Establishes the number of workers available in the distributed DataFusion cluster, their URLs, and how to connect -to them. +Establishes the number of workers available in the distributed DataFusion cluster by returning their URLs. -Each organization does networking differently, so this extension allows you to plug in a custom networking -implementation that caters to your organization's needs. +Different organizations have different needs regarding networking. Some might be using Kubernetes, some other might +be using a cloud provider solution, and this trait allows adapting Distributed DataFusion to the different scenarios. + +## [ChannelResolver](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/src/networking/channel_resolver.rs) + +Optional extension trait that allows to customize how connections are established to workers. Given one of the +URLs returned by the `WorkerResolver`, it builds an Arrow Flight client ready for serving queries. ## [NetworkBoundary](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/src/distributed_planner/network_boundary.rs) diff --git a/docs/source/user-guide/getting-started.md b/docs/source/user-guide/getting-started.md index ead5275b..c7c52bb8 100644 --- a/docs/source/user-guide/getting-started.md +++ b/docs/source/user-guide/getting-started.md @@ -13,22 +13,21 @@ ships: ```rs let state = SessionStateBuilder::new() -+ .with_distributed_channel_resolver(my_custom_channel_resolver) ++ .with_distributed_worker_resolver(my_custom_worker_resolver) + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) .build(); ``` -And the `my_custom_channel_resolver` variable should be an implementation of -the [ChannelResolver](https://github.com/datafusion-contrib/datafusion-distributed/blob/6d014eaebd809bcbe676823698838b2c83d93900/src/channel_resolver_ext.rs#L57-L57) +And the `my_custom_worker_resolver` variable should be an implementation of +the [WorkerResolverResolver](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/src/networking/worker_resolver.rs) trait, which tells Distributed DataFusion how to connect to other workers in the cluster. A very basic example of such an implementation that resolves workers in the localhost machine is: ```rust #[derive(Clone)] -struct LocalhostChannelResolver { +struct LocalhostWorkerResolver { ports: Vec, - cached: DashMap>, } #[async_trait] @@ -36,29 +35,12 @@ impl ChannelResolver for LocalhostChannelResolver { fn get_urls(&self) -> Result, DataFusionError> { Ok(self.ports.iter().map(|port| Url::parse(&format!("http://localhost:{port}")).unwrap()).collect()) } - - async fn get_flight_client_for_url( - &self, - url: &Url, - ) -> Result, DataFusionError> { - match self.cached.entry(url.clone()) { - Entry::Occupied(v) => Ok(v.get().clone()), - Entry::Vacant(v) => { - let channel = Channel::from_shared(url.to_string()) - .unwrap() - .connect_lazy(); - let channel = FlightServiceClient::new(BoxCloneSyncChannel::new(channel)); - v.insert(channel.clone()); - Ok(channel) - } - } - } } ``` > NOTE: This example is not production-ready and is meant to showcase the basic concepts of the library. -This `ChannelResolver` implementation should resolve URLs of Distributed DataFusion Arrow Flight servers, and it's +This `WorkerResolver` implementation should resolve URLs of Distributed DataFusion Arrow Flight servers, and it's also the user of this library's responsibility to spawn a Tonic server that exposes the Arrow Flight service. A basic example of such a server is: @@ -66,18 +48,7 @@ A basic example of such a server is: ```rust #[tokio::main] async fn main() -> Result<(), Box> { - let my_custom_channel_resolver = todo!(); - - let endpoint = ArrowFlightEndpoint::try_new(move |ctx: DistributedSessionBuilderContext| { - let my_custom_channel_resolver = my_custom_channel_resolver.clone(); - async move { - Ok(SessionStateBuilder::new() - .with_runtime_env(ctx.runtime_env) - .with_distributed_channel_resolver(my_custom_channel_resolver) - .with_default_features() - .build()) - } - })?; + let endpoint = ArrowFlightEndpoint::default(); Server::builder() .add_service(endpoint.into_flight_server()) @@ -92,6 +63,7 @@ async fn main() -> Result<(), Box> { The next two sections of this guide will walk you through tailoring the library's traits to your own needs: +- [Build your own WorkerResolver](worker-resolver.md) - [Build your own ChannelResolver](channel-resolver.md) - [Build your own TaskEstimator](task-estimator.md) - [Build your own distributed DataFusion Arrow Flight endpoint](arrow-flight-endpoint.md) diff --git a/docs/source/user-guide/index.md b/docs/source/user-guide/index.md index c41b8482..74cb24d3 100644 --- a/docs/source/user-guide/index.md +++ b/docs/source/user-guide/index.md @@ -9,7 +9,8 @@ your own distributed DataFusion cluster. - [Concepts](concepts.md) - [Getting Started](getting-started.md) +- [Building a WorkerResolver](worker-resolver.md) +- [Building an Arrow Flight endpoint](arrow-flight-endpoint.md) - [Building a ChannelResolver](channel-resolver.md) - [Building a TaskEstimator](task-estimator.md) -- [Building an Arrow Flight endpoint](arrow-flight-endpoint.md) - [How a distributed plan is built](how-a-distributed-plan-is-built.md) diff --git a/docs/source/user-guide/task-estimator.md b/docs/source/user-guide/task-estimator.md index 736f2fcc..77eaacb1 100644 --- a/docs/source/user-guide/task-estimator.md +++ b/docs/source/user-guide/task-estimator.md @@ -1,3 +1,30 @@ # Building a TaskEstimator -> WARNING: under construction +A `TaskEstimator` is a trait that tells the distributed planner how many distributed tasks should be used in the +different stages of the plan. + +The number of tasks is assigned to the different stages in a bottom-to-top fashion. You can refer the +[Plan Annotation docs](https://github.com/datafusion-contrib/datafusion-distributed/blob/75b4e73e9052c6596b9d1744ce2bdfa6cbc010d3/src/distributed_planner/plan_annotator.rs) +for an explanation on how this works. A `TaskEstimator` is what hints this process how many tasks should be +used. + +There is a default implementation that acts on `DataSourceExec` nodes baked by `FileScanConfig`s, however, as a user, +you may want to provide your own `TaskEstimator` implementation for your own `ExecutionPlan`s. + +## Providing your own TaskEstimator + +Providing a `TaskEstimator` allows you to do two things: + +1. Tell the distributed planner how many tasks should be used for your own `ExecutionPlan`s. +2. Tell the distributed planner how to "scale up" your `ExecutionPlan` in order to account for it running in + multiple distributed tasks. + +Note that if your custom nodes are going to run distributed, you need to account for it at execution time. +If you build a `TaskEstimator` that tells the distributed planner that your node should run in N tasks, then +you need to react to the presence of a +[DistributedTaskCtx](https://github.com/datafusion-contrib/datafusion-distributed/blob/75b4e73e9052c6596b9d1744ce2bdfa6cbc010d3/src/stage.rs#L137-L137) +during execution. + +There's an example of how to do that in the `examples/` folder: + +- TODO: link to example diff --git a/docs/source/user-guide/worker-resolver.md b/docs/source/user-guide/worker-resolver.md new file mode 100644 index 00000000..ee23cdbb --- /dev/null +++ b/docs/source/user-guide/worker-resolver.md @@ -0,0 +1,72 @@ +# Building a WorkerResolver + +A `WorkerResolver` tells distributed DataFusion the location (URLs) of your worker nodes. This information is +used in two different places: + +1. For planning: during distributed planning, the amount of worker nodes available (essentially, `Vec`.length()), + is used for determined how to scale up the plan. The distributed planner will not use more tasks per stage than the + amount of workers the cluster has. +2. Right before execution: each task in a distributed plan contains slots that the `DistributedExec` node needs + to fill right before execution (with the URLs of the workers as updated as possible). + +You need to pass your own `WorkerResolver` to DataFusion's `SessionStateBuilder` so that it's available in the +`SesionContext`: + +```rust +struct CustomWorkerResolver; + +#[async_trait] +impl WorkerResolver for CustomWorkerResolver { + fn get_urls(&self) -> Result, DataFusionError> { + todo!() + } +} + +async fn main() { + let state = SessionStateBuilder::new() + .with_distributed_worker_resolver(CustomWorkerResolver) + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + .build(); +} +``` + +> NOTE: It's not necessary to pass this to the Arrow Flight endpoint session builder. + +## Static WorkerResolver + +This is the simplest thing you can do, although it will not fit some common use cases. An example of this can be +seen +in [localhost_worker.rs](https://github.com/datafusion-contrib/datafusion-distributed/blob/fad9fa222d65b7d2ddae92fbc20082b5c434e4ff/examples/localhost_run.rs) +example: + +```rust +#[derive(Clone)] +struct LocalhostChannelResolver { + ports: Vec, +} + +#[async_trait] +impl WorkerResolver for LocalhostChannelResolver { + fn get_urls(&self) -> Result, DataFusionError> { + Ok(self + .ports + .iter() + .map(|port| Url::parse(&format!("http://localhost:{port}")).unwrap()) + .collect()) + } +} +``` + +## Dynamic WorkerResolver + +In a typical setup, you might have different pods running in a Kubernetes cluster, or you might be using a cloud +provider for hosting your Distributed DataFusion workers. + +It's up to you to decide how the URLs should be resolved. One important implementation note is: + +- Retrieving the worker URLs needs to be synchronous (as planning is synchronous), and therefore, you'll likely + need to spawn a background tasks that periodically refreshes the list of available URLs. + +A good example can be found +in https://github.com/datafusion-contrib/datafusion-distributed/blob/main/benchmarks/cdk/bin/worker.rs, +where a cluster of AWS EC2 machines are discovered identified by tags with the AWS Rust SDK. \ No newline at end of file From 84243d3c92e50fa5e5e78e3a15bcae3e8fcfa7ac Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Mon, 29 Dec 2025 14:48:28 +0100 Subject: [PATCH 09/27] Apply TaskEstimator on all nodes (#271) * Split channel resolver in two * Simplify WorkerResolverExtension and ChannelResolverExtension * Add default builder to ArrowFlightEndpoint * Add some docs * Listen to clippy * Split get_flight_client_for_url in two * Fix conflicts * Remove unnecessary channel resolver * Improve WorkerResolver docs * Use one ChannelResolver per runtime * Improve error reporting on client connection failure * Add a from_session_builder method for constructing an InMemoryChannelResolver * Add ChannelResolver and WorkerResolver default implementations for Arcs * Make TPC-DS tests use DataFusion test dataset * Remove non-working in-memory option from benchmarks * Remove unnecessary utils folder * Refactor benchmark folder * Rename to prepare_tpch.rs * Adapt benchmarks for TPC-DS * Update benchmarks README.md * Fix conflicts * Use default session state builder * Add ChildrenIsolatorUnionExec * Add proto serde for ChildrenIsolatorUnionExec * Wire up ChildrenIsolatorUnionExec to planner * Add integration tests for distributed UNIONs * Skip query 72 in TPC-DS benchmarks * Allow setting children isolator unions * Allow passing multiple queries in benchmarks * Extend TaskEstimator API to also be applicable to intermediate nodes --- src/distributed_planner/plan_annotator.rs | 118 ++++++++++++++++++++-- src/distributed_planner/task_estimator.rs | 117 ++++++++++++--------- 2 files changed, 178 insertions(+), 57 deletions(-) diff --git a/src/distributed_planner/plan_annotator.rs b/src/distributed_planner/plan_annotator.rs index 9704f646..87b93671 100644 --- a/src/distributed_planner/plan_annotator.rs +++ b/src/distributed_planner/plan_annotator.rs @@ -138,6 +138,7 @@ fn _annotate_plan( ) -> Result { use TaskCountAnnotation::*; let d_cfg = DistributedConfig::from_config_options(cfg)?; + let estimator = &d_cfg.__private_task_estimator; let n_workers = d_cfg.__private_worker_resolver.0.get_urls()?.len().max(1); let annotated_children = plan @@ -150,8 +151,7 @@ fn _annotate_plan( // This is a leaf node, maybe a DataSourceExec, or maybe something else custom from the // user. We need to estimate how many tasks are needed for this leaf node, and we'll take // this decision into account when deciding how many tasks will be actually used. - let estimator = &d_cfg.__private_task_estimator; - if let Some(estimate) = estimator.tasks_for_leaf_node(&plan, cfg) { + if let Some(estimate) = estimator.task_estimation(&plan, cfg) { return Ok(AnnotatedPlan { plan, children: Vec::new(), @@ -170,7 +170,9 @@ fn _annotate_plan( } } - let mut task_count = Desired(1); + let mut task_count = estimator + .task_estimation(&plan, cfg) + .map_or(Desired(1), |v| v.task_count); if d_cfg.children_isolator_unions && plan.as_any().is::() { // Unions have the chance to decide how many tasks they should run on. If there's a union // with a bunch of children, the user might want to increase parallelism and increase the @@ -341,11 +343,11 @@ mod tests { use super::*; use crate::test_utils::in_memory_channel_resolver::InMemoryWorkerResolver; use crate::test_utils::parquet::register_parquet_tables; - use crate::{DistributedExt, assert_snapshot}; + use crate::{DistributedExt, TaskEstimation, assert_snapshot}; use datafusion::execution::SessionStateBuilder; + use datafusion::physical_plan::filter::FilterExec; use datafusion::prelude::{SessionConfig, SessionContext}; use itertools::Itertools; - /* schema for the "weather" table MinTemp [type=DOUBLE] [repetitiontype=OPTIONAL] @@ -584,16 +586,116 @@ mod tests { ") } + #[tokio::test] + async fn test_intermediate_task_estimator() { + let query = r#" + SELECT DISTINCT "RainToday" FROM weather + "#; + let annotated = sql_to_annotated_with_estimator(query, |_: &RepartitionExec| { + Some(TaskEstimation::maximum(1)) + }) + .await; + assert_snapshot!(annotated, @r" + AggregateExec: task_count=Desired(1) + CoalesceBatchesExec: task_count=Desired(1), required_network_boundary=Shuffle + RepartitionExec: task_count=Maximum(1) + RepartitionExec: task_count=Maximum(1) + AggregateExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + ") + } + + #[tokio::test] + async fn test_union_all_limited_by_intermediate_estimator() { + let query = r#" + SELECT "MinTemp" FROM weather WHERE "RainToday" = 'yes' + UNION ALL + SELECT "MaxTemp" FROM weather WHERE "RainToday" = 'no' + "#; + let annotated = sql_to_annotated_with_estimator(query, |_: &FilterExec| { + Some(TaskEstimation::maximum(1)) + }) + .await; + assert_snapshot!(annotated, @r" + ChildrenIsolatorUnionExec: task_count=Desired(2) + CoalesceBatchesExec: task_count=Maximum(1) + FilterExec: task_count=Maximum(1) + RepartitionExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + ProjectionExec: task_count=Maximum(1) + CoalesceBatchesExec: task_count=Maximum(1) + FilterExec: task_count=Maximum(1) + RepartitionExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + ") + } + + #[allow(clippy::type_complexity)] + struct CallbackEstimator { + f: Arc Option + Send + Sync>, + } + + impl CallbackEstimator { + fn new( + f: impl Fn(&T) -> Option + Send + Sync + 'static, + ) -> Self { + let f = Arc::new(move |plan: &dyn ExecutionPlan| -> Option { + if let Some(plan) = plan.as_any().downcast_ref::() { + f(plan) + } else { + None + } + }); + Self { f } + } + } + + impl TaskEstimator for CallbackEstimator { + fn task_estimation( + &self, + plan: &Arc, + _: &ConfigOptions, + ) -> Option { + (self.f)(plan.as_ref()) + } + + fn scale_up_leaf_node( + &self, + _: &Arc, + _: usize, + _: &ConfigOptions, + ) -> Option> { + None + } + } + async fn sql_to_annotated(query: &str) -> String { + sql_to_annotated_with_options(query, move |b| b).await + } + + async fn sql_to_annotated_with_estimator( + query: &str, + estimator: impl Fn(&T) -> Option + Send + Sync + 'static, + ) -> String { + sql_to_annotated_with_options(query, move |b| { + b.with_distributed_task_estimator(CallbackEstimator::new(estimator)) + }) + .await + } + + async fn sql_to_annotated_with_options( + query: &str, + f: impl FnOnce(SessionStateBuilder) -> SessionStateBuilder, + ) -> String { let config = SessionConfig::new() .with_target_partitions(4) .with_information_schema(true); - let state = SessionStateBuilder::new() + let state = f(SessionStateBuilder::new() .with_default_features() .with_config(config) - .with_distributed_worker_resolver(InMemoryWorkerResolver::new(4)) - .build(); + .with_distributed_worker_resolver(InMemoryWorkerResolver::new(4))) + .build(); let ctx = SessionContext::new_with_state(state); let mut queries = query.split(";").collect_vec(); diff --git a/src/distributed_planner/task_estimator.rs b/src/distributed_planner/task_estimator.rs index b32b7dbf..2bf74ab0 100644 --- a/src/distributed_planner/task_estimator.rs +++ b/src/distributed_planner/task_estimator.rs @@ -5,6 +5,7 @@ use datafusion::config::ConfigOptions; use datafusion::datasource::physical_plan::FileScanConfig; use datafusion::physical_plan::ExecutionPlan; use datafusion::prelude::SessionConfig; +use delegate::delegate; use std::collections::HashSet; use std::fmt::Debug; use std::sync::Arc; @@ -58,6 +59,34 @@ pub struct TaskEstimation { pub task_count: TaskCountAnnotation, } +impl TaskEstimation { + /// Tells the distributed planner that the evaluated stage can have **at maximum** the provided + /// number of tasks, setting a hard upper limit. + /// + /// Returning `TaskEstimation::maximum(1)` tells the distributed planner that the evaluated + /// stage cannot be distributed. + /// + /// Even if a `TaskEstimation::maximum(N)` is provided, any other node in the same stage + /// providing a value of `TaskEstimation::maximum(M)` where `M` < `N` will have preference. + pub fn maximum(value: usize) -> Self { + TaskEstimation { + task_count: TaskCountAnnotation::Maximum(value), + } + } + + /// Tells the distributed planner that the evaluated can **optimally** have the provided + /// number of tasks, setting a soft task count hint that can be overridden by others. + /// + /// The provided `TaskEstimation::desired(N)` can be overridden by: + /// - Other nodes providing a `TaskEstimation::desired(M)` where `M` > `N`. + /// - Any other node providing a `TaskEstimation::maximum(M)` where `M` can be anything. + pub fn desired(value: usize) -> Self { + TaskEstimation { + task_count: TaskCountAnnotation::Desired(value), + } + } +} + /// Given a leaf node, provides an estimation about how many tasks should be used in the /// stage containing it, and if the leaf node should be replaced by some other. /// @@ -66,14 +95,19 @@ pub struct TaskEstimation { /// count calculated based on whether lower stages are reducing the cardinality of the data /// or increasing it. pub trait TaskEstimator { - /// Function applied to leaf nodes that returns a [TaskEstimation] hinting how many - /// tasks should be used in the [Stage] containing that leaf node. + /// Function applied to each node that returns a [TaskEstimation] hinting how many + /// tasks should be used in the [Stage] containing that node. + /// + /// All the [TaskEstimator] registered in the session will be applied to the node + /// until one returns an estimation. + /// /// - /// All the [TaskEstimator] registered in the session will be applied to the leaf node - /// until one returns an estimation. If no estimation is return from any of the - /// [TaskEstimator]s, then `Maximum(1)` is returned, hinting the distributed planner to not - /// distribute the stage containing that node. - fn tasks_for_leaf_node( + /// If no estimation is returned from any of the registered [TaskEstimator]s, then: + /// - If the node is a leaf node,`Maximum(1)` is assumed, hinting the distributed planner + /// that the leaf node cannot be distributed across tasks. + /// - If the node is a normal node in the plan, then the maximum task count from its children + /// is inherited. + fn task_estimation( &self, plan: &Arc, cfg: &ConfigOptions, @@ -91,14 +125,18 @@ pub trait TaskEstimator { } impl TaskEstimator for usize { - fn tasks_for_leaf_node( + fn task_estimation( &self, - _: &Arc, + inputs: &Arc, _: &ConfigOptions, ) -> Option { - Some(TaskEstimation { - task_count: TaskCountAnnotation::Desired(*self), - }) + if inputs.children().is_empty() { + Some(TaskEstimation { + task_count: TaskCountAnnotation::Desired(*self), + }) + } else { + None + } } fn scale_up_leaf_node( @@ -112,40 +150,20 @@ impl TaskEstimator for usize { } impl TaskEstimator for Arc { - fn tasks_for_leaf_node( - &self, - plan: &Arc, - cfg: &ConfigOptions, - ) -> Option { - self.as_ref().tasks_for_leaf_node(plan, cfg) - } - - fn scale_up_leaf_node( - &self, - plan: &Arc, - task_count: usize, - cfg: &ConfigOptions, - ) -> Option> { - self.as_ref().scale_up_leaf_node(plan, task_count, cfg) + delegate! { + to self.as_ref() { + fn task_estimation(&self, plan: &Arc, cfg: &ConfigOptions) -> Option; + fn scale_up_leaf_node(&self, plan: &Arc, task_count: usize, cfg: &ConfigOptions) -> Option>; + } } } impl TaskEstimator for Arc { - fn tasks_for_leaf_node( - &self, - plan: &Arc, - cfg: &ConfigOptions, - ) -> Option { - self.as_ref().tasks_for_leaf_node(plan, cfg) - } - - fn scale_up_leaf_node( - &self, - plan: &Arc, - task_count: usize, - cfg: &ConfigOptions, - ) -> Option> { - self.as_ref().scale_up_leaf_node(plan, task_count, cfg) + delegate! { + to self.as_ref() { + fn task_estimation(&self, plan: &Arc, cfg: &ConfigOptions) -> Option; + fn scale_up_leaf_node(&self, plan: &Arc, task_count: usize, cfg: &ConfigOptions) -> Option>; + } } } @@ -177,15 +195,16 @@ pub(crate) fn set_distributed_task_estimator( struct FileScanConfigTaskEstimator; impl TaskEstimator for FileScanConfigTaskEstimator { - fn tasks_for_leaf_node( + fn task_estimation( &self, plan: &Arc, cfg: &ConfigOptions, ) -> Option { - let d_cfg = cfg.extensions.get::()?; let dse: &DataSourceExec = plan.as_any().downcast_ref()?; let file_scan: &FileScanConfig = dse.data_source().as_any().downcast_ref()?; + let d_cfg = cfg.extensions.get::()?; + // Count how many distinct files we have in the FileScanConfig. Each file in each // file group is a PartitionedFile rather than a full file, so it's possible that // many entries refer to different chunks of the same physical file. By keeping a @@ -244,13 +263,13 @@ pub(crate) struct CombinedTaskEstimator { } impl TaskEstimator for CombinedTaskEstimator { - fn tasks_for_leaf_node( + fn task_estimation( &self, plan: &Arc, cfg: &ConfigOptions, ) -> Option { for estimator in &self.user_provided { - if let Some(result) = estimator.tasks_for_leaf_node(plan, cfg) { + if let Some(result) = estimator.task_estimation(plan, cfg) { return Some(result); } } @@ -258,7 +277,7 @@ impl TaskEstimator for CombinedTaskEstimator { // a chance of providing an estimation. // If none of the user-provided returned an estimation, the default ones are used. for default_estimator in [&FileScanConfigTaskEstimator as &dyn TaskEstimator] { - if let Some(result) = default_estimator.tasks_for_leaf_node(plan, cfg) { + if let Some(result) = default_estimator.task_estimation(plan, cfg) { return Some(result); } } @@ -348,7 +367,7 @@ mod tests { ..Default::default() }; cfg.extensions.insert(f(d_cfg)); - self.tasks_for_leaf_node(&node, &cfg) + self.task_estimation(&node, &cfg) .unwrap() .task_count .as_usize() @@ -370,7 +389,7 @@ mod tests { } impl, &ConfigOptions) -> Option> TaskEstimator for F { - fn tasks_for_leaf_node( + fn task_estimation( &self, plan: &Arc, cfg: &ConfigOptions, From f7318aafedc3bd4544f99b80e94b6b396126abf0 Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Tue, 30 Dec 2025 11:43:21 +0100 Subject: [PATCH 10/27] Rename all arrow flight endpoint references to "Worker" (#274) --- README.md | 6 +- benchmarks/cdk/bin/worker.rs | 15 +++-- benchmarks/src/run.rs | 4 +- docs/source/user-guide/channel-resolver.md | 6 +- docs/source/user-guide/getting-started.md | 9 +-- docs/source/user-guide/index.md | 2 +- docs/source/user-guide/worker-resolver.md | 10 +-- .../{arrow-flight-endpoint.md => worker.md} | 38 ++++++------ examples/in_memory_cluster.rs | 15 ++--- examples/localhost.md | 4 +- examples/localhost_worker.rs | 4 +- src/distributed_ext.rs | 62 ++++++++++--------- src/execution_plans/network_coalesce.rs | 6 +- src/execution_plans/network_shuffle.rs | 2 +- src/flight_service/do_get.rs | 12 ++-- src/flight_service/mod.rs | 8 +-- src/flight_service/session_builder.rs | 57 +++++++++-------- src/flight_service/{service.rs => worker.rs} | 44 ++++++++----- src/lib.rs | 5 +- src/stage.rs | 22 +++---- src/test_utils/in_memory_channel_resolver.rs | 10 +-- src/test_utils/localhost.rs | 12 ++-- tests/custom_config_extension.rs | 6 +- tests/custom_extension_codec.rs | 8 +-- tests/error_propagation.rs | 6 +- tests/introspection.rs | 19 +++--- tests/stateful_execution_plan.rs | 12 ++-- tests/udfs.rs | 8 +-- 28 files changed, 199 insertions(+), 213 deletions(-) rename docs/source/user-guide/{arrow-flight-endpoint.md => worker.md} (59%) rename src/flight_service/{service.rs => worker.rs} (85%) diff --git a/README.md b/README.md index c1de5f79..f7e6cdb7 100644 --- a/README.md +++ b/README.md @@ -35,10 +35,10 @@ https://datafusion-contrib.github.io/datafusion-distributed There are some runnable examples showcasing how to provide a localhost implementation for Distributed DataFusion in [examples/](examples): -- [localhost_worker.rs](examples/localhost_worker.rs): code that spawns an Arrow Flight Endpoint listening for physical +- [localhost_worker.rs](examples/localhost_worker.rs): code that spawns a Worker listening for physical plans over the network. -- [localhost_run.rs](examples/localhost_run.rs): code that distributes a query across the spawned Arrow Flight Endpoints - and executes it. +- [localhost_run.rs](examples/localhost_run.rs): code that distributes a query across the spawned Workers and executes + it. The integration tests also provide an idea about how to use the library and what can be achieved with it: diff --git a/benchmarks/cdk/bin/worker.rs b/benchmarks/cdk/bin/worker.rs index 30bd5d50..ab3e398d 100644 --- a/benchmarks/cdk/bin/worker.rs +++ b/benchmarks/cdk/bin/worker.rs @@ -9,8 +9,8 @@ use datafusion::execution::SessionStateBuilder; use datafusion::physical_plan::execute_stream; use datafusion::prelude::SessionContext; use datafusion_distributed::{ - ArrowFlightEndpoint, DistributedExt, DistributedPhysicalOptimizerRule, - DistributedSessionBuilderContext, WorkerResolver, display_plan_ascii, + DistributedExt, DistributedPhysicalOptimizerRule, Worker, WorkerQueryContext, WorkerResolver, + display_plan_ascii, }; use futures::{StreamExt, TryFutureExt}; use log::{error, info, warn}; @@ -72,12 +72,11 @@ async fn main() -> Result<(), Box> { .build(); let ctx = SessionContext::from(state); - let arrow_flight_endpoint = - ArrowFlightEndpoint::from_session_builder(move |ctx: DistributedSessionBuilderContext| { - let s3 = s3.clone(); - let s3_url = s3_url.clone(); - async move { Ok(ctx.builder.with_object_store(&s3_url, s3).build()) } - }); + let arrow_flight_endpoint = Worker::from_session_builder(move |ctx: WorkerQueryContext| { + let s3 = s3.clone(); + let s3_url = s3_url.clone(); + async move { Ok(ctx.builder.with_object_store(&s3_url, s3).build()) } + }); let http_server = axum::serve( listener, Router::new().route( diff --git a/benchmarks/src/run.rs b/benchmarks/src/run.rs index 60b91dae..cdbff0a3 100644 --- a/benchmarks/src/run.rs +++ b/benchmarks/src/run.rs @@ -31,7 +31,7 @@ use datafusion::prelude::*; use datafusion_distributed::test_utils::localhost::LocalHostWorkerResolver; use datafusion_distributed::test_utils::{clickbench, tpcds, tpch}; use datafusion_distributed::{ - ArrowFlightEndpoint, DistributedExt, DistributedPhysicalOptimizerRule, NetworkBoundaryExt, + DistributedExt, DistributedPhysicalOptimizerRule, NetworkBoundaryExt, Worker, }; use log::info; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -185,7 +185,7 @@ impl RunOpt { let incoming = tokio_stream::wrappers::TcpListenerStream::new(listener); Ok::<_, Box>( Server::builder() - .add_service(ArrowFlightEndpoint::default().into_flight_server()) + .add_service(Worker::default().into_flight_server()) .serve_with_incoming(incoming) .await?, ) diff --git a/docs/source/user-guide/channel-resolver.md b/docs/source/user-guide/channel-resolver.md index 4375c88b..c984459b 100644 --- a/docs/source/user-guide/channel-resolver.md +++ b/docs/source/user-guide/channel-resolver.md @@ -13,7 +13,7 @@ points: - You will need to provide your own implementation in two places: - in the `SessionContext` that handles your queries. - - while instantiating the `ArrowFlightEndpoint` with the `from_session_builder()` constructor. + - while instantiating the `Worker` with the `from_session_builder()` constructor. - If you decide to build it from scratch, make sure that Arrow Flight clients are reused across request rather than always building new ones. - You can use this library's `DefaultChannelResolver` as a backbone for your own implementation. @@ -44,10 +44,10 @@ async fn main() { .with_distributed_channel_resolver(channel_resolver.clone()) .build(); - let endpoint = ArrowFlightEndpoint::from_session_builder(|ctx: DistributedSessionBuilderContext| { + // ... and here for each query the Worker handles. + let endpoint = Worker::from_session_builder(move |ctx: WorkerQueryContext| { let channel_resolver = channel_resolver.clone(); async move { - // ... and here for each query that ArrowFlightEndpoint handles. Ok(ctx.builder.with_distributed_channel_resolver(channel_resolver).build()) } }); diff --git a/docs/source/user-guide/getting-started.md b/docs/source/user-guide/getting-started.md index c7c52bb8..0682180c 100644 --- a/docs/source/user-guide/getting-started.md +++ b/docs/source/user-guide/getting-started.md @@ -40,15 +40,16 @@ impl ChannelResolver for LocalhostChannelResolver { > NOTE: This example is not production-ready and is meant to showcase the basic concepts of the library. -This `WorkerResolver` implementation should resolve URLs of Distributed DataFusion Arrow Flight servers, and it's -also the user of this library's responsibility to spawn a Tonic server that exposes the Arrow Flight service. +This `WorkerResolver` implementation should resolve URLs of Distributed DataFusion workers, and it's +also the user of this library's responsibility to spawn a Tonic server that exposes the worker as an Arrow Flight +service using Tonic. A basic example of such a server is: ```rust #[tokio::main] async fn main() -> Result<(), Box> { - let endpoint = ArrowFlightEndpoint::default(); + let endpoint = Worker::default(); Server::builder() .add_service(endpoint.into_flight_server()) @@ -66,7 +67,7 @@ The next two sections of this guide will walk you through tailoring the library' - [Build your own WorkerResolver](worker-resolver.md) - [Build your own ChannelResolver](channel-resolver.md) - [Build your own TaskEstimator](task-estimator.md) -- [Build your own distributed DataFusion Arrow Flight endpoint](arrow-flight-endpoint.md) +- [Build your own distributed DataFusion Worker](worker.md) Here are some other resources in the codebase: diff --git a/docs/source/user-guide/index.md b/docs/source/user-guide/index.md index 74cb24d3..f4d4022d 100644 --- a/docs/source/user-guide/index.md +++ b/docs/source/user-guide/index.md @@ -10,7 +10,7 @@ your own distributed DataFusion cluster. - [Concepts](concepts.md) - [Getting Started](getting-started.md) - [Building a WorkerResolver](worker-resolver.md) -- [Building an Arrow Flight endpoint](arrow-flight-endpoint.md) +- [Spawning a Worker](worker.md) - [Building a ChannelResolver](channel-resolver.md) - [Building a TaskEstimator](task-estimator.md) - [How a distributed plan is built](how-a-distributed-plan-is-built.md) diff --git a/docs/source/user-guide/worker-resolver.md b/docs/source/user-guide/worker-resolver.md index ee23cdbb..8af97362 100644 --- a/docs/source/user-guide/worker-resolver.md +++ b/docs/source/user-guide/worker-resolver.md @@ -30,13 +30,13 @@ async fn main() { } ``` -> NOTE: It's not necessary to pass this to the Arrow Flight endpoint session builder. +> NOTE: It's not necessary to pass this to the Worker session builder. ## Static WorkerResolver This is the simplest thing you can do, although it will not fit some common use cases. An example of this can be -seen -in [localhost_worker.rs](https://github.com/datafusion-contrib/datafusion-distributed/blob/fad9fa222d65b7d2ddae92fbc20082b5c434e4ff/examples/localhost_run.rs) +seen in the +[localhost_worker.rs](https://github.com/datafusion-contrib/datafusion-distributed/blob/fad9fa222d65b7d2ddae92fbc20082b5c434e4ff/examples/localhost_run.rs) example: ```rust @@ -65,8 +65,8 @@ provider for hosting your Distributed DataFusion workers. It's up to you to decide how the URLs should be resolved. One important implementation note is: - Retrieving the worker URLs needs to be synchronous (as planning is synchronous), and therefore, you'll likely - need to spawn a background tasks that periodically refreshes the list of available URLs. + need to spawn background tasks that periodically refresh the list of available URLs. A good example can be found in https://github.com/datafusion-contrib/datafusion-distributed/blob/main/benchmarks/cdk/bin/worker.rs, -where a cluster of AWS EC2 machines are discovered identified by tags with the AWS Rust SDK. \ No newline at end of file +where a cluster of AWS EC2 machines is discovered identified by tags with the AWS Rust SDK. \ No newline at end of file diff --git a/docs/source/user-guide/arrow-flight-endpoint.md b/docs/source/user-guide/worker.md similarity index 59% rename from docs/source/user-guide/arrow-flight-endpoint.md rename to docs/source/user-guide/worker.md index 1c8192f8..573d36c4 100644 --- a/docs/source/user-guide/arrow-flight-endpoint.md +++ b/docs/source/user-guide/worker.md @@ -1,26 +1,26 @@ -# Building an Arrow Flight Endpoint +# Spawn a Worker -An `ArrowFlightEndpoint` is a gRPC server that implements the Arrow Flight protocol for distributed query execution. +A `Worker` is a gRPC server that implements the Arrow Flight protocol for distributed query execution. Worker nodes run these endpoints to receive execution plans, execute them, and stream results back. ## Overview -The `ArrowFlightEndpoint` is the core worker component in Distributed DataFusion. It: +The `Worker` is the core worker component in Distributed DataFusion. It: - Receives serialized execution plans via Arrow Flight's `do_get` method - Deserializes plans using protobuf and user-provided codecs - Executes plans using the local DataFusion runtime - Streams results back as Arrow record batches through the gRPC Arrow Flight interface -## Creating an ArrowFlightEndpoint +## Launching the Arrow Flight server -The `Default` implementation of the `ArrowFlightEndpoint` should satisfy most basic use cases +The `Default` implementation of the `Worker` should satisfy most basic use cases ```rust -use datafusion_distributed::{ArrowFlightEndpoint}; +use datafusion_distributed::Worker; async fn main() { - let endpoint = ArrowFlightEndpoint::default(); + let endpoint = Worker::default(); Server::builder() .add_service(endpoint.into_flight_server()) @@ -31,13 +31,13 @@ async fn main() { } ``` -If you are using DataFusion though, it's very likely that you have your own custom UDFs, execution nodes, config options -etc... +If you are using DataFusion, though, it's very likely that you have your own custom UDFs, execution nodes, config +options, etc... -You'll need to tell the `ArrowFlightEndpoint` how to build your DataFusion sessions: +You'll need to tell the `Worker` how to build your DataFusion sessions: ```rust -async fn build_state(ctx: DistributedSessionBuilderContext) -> Result { +async fn build_state(ctx: WorkerQueryContext) -> Result { Ok(ctx .builder .with_scalar_functions(vec![your_custom_udf()]) @@ -45,7 +45,7 @@ async fn build_state(ctx: DistributedSessionBuilderContext) -> Result Result; } ``` -It receives a `DistributedSessionBuilderContext` containing: +It receives a `WorkerQueryContext` containing: - `SessionStateBuilder`: A pre-populated session state builder in which you can inject your custom stuff - `headers`: HTTP headers from the incoming request (useful for passing metadata) @@ -81,11 +81,11 @@ Convert the endpoint to a gRPC service and serve it: ```rust use tonic::transport::Server; -use datafusion_distributed::ArrowFlightEndpoint; +use datafusion_distributed::Worker; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; async fn main() { - let endpoint = ArrowFlightEndpoint::default(); + let endpoint = Worker::default(); Server::builder() .add_service(endpoint.into_flight_server()) diff --git a/examples/in_memory_cluster.rs b/examples/in_memory_cluster.rs index da1720ba..e5f02fdc 100644 --- a/examples/in_memory_cluster.rs +++ b/examples/in_memory_cluster.rs @@ -5,9 +5,8 @@ use datafusion::common::DataFusionError; use datafusion::execution::SessionStateBuilder; use datafusion::prelude::{ParquetReadOptions, SessionContext}; use datafusion_distributed::{ - ArrowFlightEndpoint, BoxCloneSyncChannel, ChannelResolver, DistributedExt, - DistributedPhysicalOptimizerRule, DistributedSessionBuilderContext, WorkerResolver, - create_flight_client, display_plan_ascii, + BoxCloneSyncChannel, ChannelResolver, DistributedExt, DistributedPhysicalOptimizerRule, Worker, + WorkerQueryContext, WorkerResolver, create_flight_client, display_plan_ascii, }; use futures::TryStreamExt; use hyper_util::rt::TokioIo; @@ -89,12 +88,10 @@ impl InMemoryChannelResolver { }; let this_clone = this.clone(); - let endpoint = ArrowFlightEndpoint::from_session_builder( - move |ctx: DistributedSessionBuilderContext| { - let this = this.clone(); - async move { Ok(ctx.builder.with_distributed_channel_resolver(this).build()) } - }, - ); + let endpoint = Worker::from_session_builder(move |ctx: WorkerQueryContext| { + let this = this.clone(); + async move { Ok(ctx.builder.with_distributed_channel_resolver(this).build()) } + }); tokio::spawn(async move { Server::builder() diff --git a/examples/localhost.md b/examples/localhost.md index a3172c60..497e61f4 100644 --- a/examples/localhost.md +++ b/examples/localhost.md @@ -16,7 +16,7 @@ git lfs checkout ### Spawning the workers -In two different terminals spawn two ArrowFlightEndpoints +In two different terminals spawn two workers ```shell cargo run --example localhost_worker -- 8080 @@ -26,7 +26,7 @@ cargo run --example localhost_worker -- 8080 cargo run --example localhost_worker -- 8081 ``` -The positional numeric argument is the port in which each Arrow Flight endpoint will listen to. +The positional numeric argument is the port in which each worker will listen to. ### Issuing a distributed SQL query diff --git a/examples/localhost_worker.rs b/examples/localhost_worker.rs index 52c723e6..58d329e0 100644 --- a/examples/localhost_worker.rs +++ b/examples/localhost_worker.rs @@ -1,4 +1,4 @@ -use datafusion_distributed::ArrowFlightEndpoint; +use datafusion_distributed::Worker; use std::error::Error; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use structopt::StructOpt; @@ -16,7 +16,7 @@ async fn main() -> Result<(), Box> { let args = Args::from_args(); Server::builder() - .add_service(ArrowFlightEndpoint::default().into_flight_server()) + .add_service(Worker::default().into_flight_server()) .serve(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), args.port)) .await?; diff --git a/src/distributed_ext.rs b/src/distributed_ext.rs index 4228f4cf..97bde461 100644 --- a/src/distributed_ext.rs +++ b/src/distributed_ext.rs @@ -32,7 +32,7 @@ pub trait DistributedExt: Sized { /// # use datafusion::config::ConfigExtension; /// # use datafusion::execution::{SessionState, SessionStateBuilder}; /// # use datafusion::prelude::SessionConfig; - /// # use datafusion_distributed::{DistributedExt, DistributedSessionBuilder, DistributedSessionBuilderContext}; + /// # use datafusion_distributed::{DistributedExt, WorkerSessionBuilder, WorkerQueryContext}; /// /// extensions_options! { /// pub struct CustomExtension { @@ -53,10 +53,11 @@ pub trait DistributedExt: Sized { /// .with_distributed_option_extension(my_custom_extension).unwrap() /// .build(); /// - /// async fn build_state(ctx: DistributedSessionBuilderContext) -> Result { - /// // This function can be provided to an ArrowFlightEndpoint in order to tell it how to + /// async fn build_state(ctx: WorkerQueryContext) -> Result { + /// // This function can be provided to a Worker to tell it how to /// // build sessions that retrieve the CustomExtension from gRPC metadata. - /// Ok(SessionStateBuilder::new() + /// Ok(ctx + /// .builder /// .with_distributed_option_extension_from_headers::(&ctx.headers)? /// .build()) /// } @@ -78,7 +79,7 @@ pub trait DistributedExt: Sized { /// plan. /// /// - If there was a [ConfigExtension] of the same type already present, it's updated with an - /// in-place mutation base on the headers that came over the wire. + /// in-place mutation based on the headers that came over the wire. /// - If there was no [ConfigExtension] set before, it will get added, as if /// [SessionConfig::with_option_extension] was being called. /// @@ -90,7 +91,7 @@ pub trait DistributedExt: Sized { /// # use datafusion::config::ConfigExtension; /// # use datafusion::execution::{SessionState, SessionStateBuilder}; /// # use datafusion::prelude::SessionConfig; - /// # use datafusion_distributed::{DistributedExt, DistributedSessionBuilder, DistributedSessionBuilderContext}; + /// # use datafusion_distributed::{DistributedExt, WorkerSessionBuilder, WorkerQueryContext}; /// /// extensions_options! { /// pub struct CustomExtension { @@ -111,10 +112,11 @@ pub trait DistributedExt: Sized { /// .with_distributed_option_extension(my_custom_extension).unwrap() /// .build(); /// - /// async fn build_state(ctx: DistributedSessionBuilderContext) -> Result { - /// // This function can be provided to an ArrowFlightEndpoint in order to tell it how to + /// async fn build_state(ctx: WorkerQueryContext) -> Result { + /// // This function can be provided to a Worker to tell it how to /// // build sessions that retrieve the CustomExtension from gRPC metadata. - /// Ok(SessionStateBuilder::new() + /// Ok(ctx + /// .builder /// .with_distributed_option_extension_from_headers::(&ctx.headers)? /// .build()) /// } @@ -143,7 +145,7 @@ pub trait DistributedExt: Sized { /// # use datafusion::physical_plan::ExecutionPlan; /// # use datafusion::prelude::SessionConfig; /// # use datafusion_proto::physical_plan::PhysicalExtensionCodec; - /// # use datafusion_distributed::{DistributedExt, DistributedSessionBuilderContext}; + /// # use datafusion_distributed::{DistributedExt, WorkerQueryContext}; /// /// #[derive(Debug)] /// struct CustomExecCodec; @@ -162,8 +164,8 @@ pub trait DistributedExt: Sized { /// .with_distributed_user_codec(CustomExecCodec) /// .build(); /// - /// async fn build_state(ctx: DistributedSessionBuilderContext) -> Result { - /// // This function can be provided to an ArrowFlightEndpoint in order to tell it how to + /// async fn build_state(ctx: WorkerQueryContext) -> Result { + /// // This function can be provided to a Worker to tell it how to /// // encode/decode CustomExec nodes. /// Ok(SessionStateBuilder::new() /// .with_distributed_user_codec(CustomExecCodec) @@ -187,9 +189,8 @@ pub trait DistributedExt: Sized { /// nodes in the cluster. When running in distributed mode, setting a [WorkerResolver] is required. /// /// Even if this is required to be present in the [SessionContext] that first initiates and - /// plans the query, it's not necessary to be present in a worker's ArrowFlightEndpoint session - /// state builder, as no planning happens there, and the WorkerResolver::get_urls method is - /// only called during planning. + /// plans the query, it's not necessary to be present in a Worker's session state builder, + /// as no planning happens there. /// /// Example: /// @@ -201,7 +202,7 @@ pub trait DistributedExt: Sized { /// # use datafusion::prelude::SessionConfig; /// # use url::Url; /// # use std::sync::Arc; - /// # use datafusion_distributed::{BoxCloneSyncChannel, WorkerResolver, DistributedExt, DistributedPhysicalOptimizerRule, DistributedSessionBuilderContext}; + /// # use datafusion_distributed::{BoxCloneSyncChannel, WorkerResolver, DistributedExt, DistributedPhysicalOptimizerRule, WorkerQueryContext}; /// /// struct CustomWorkerResolver; /// @@ -233,7 +234,7 @@ pub trait DistributedExt: Sized { /// This is what tells Distributed DataFusion how to build an Arrow Flight client out of a worker URL. /// - /// There's a default implementation of this that caches the Arrow Flight client instances so that there's + /// There's a default implementation that caches the Arrow Flight client instances so that there's /// only one per URL, but users can decide to override that behavior in favor of their own solution. /// /// Example: @@ -246,7 +247,7 @@ pub trait DistributedExt: Sized { /// # use datafusion::prelude::SessionConfig; /// # use url::Url; /// # use std::sync::Arc; - /// # use datafusion_distributed::{BoxCloneSyncChannel, ChannelResolver, DistributedExt, DistributedPhysicalOptimizerRule, DistributedSessionBuilderContext}; + /// # use datafusion_distributed::{BoxCloneSyncChannel, ChannelResolver, DistributedExt, DistributedPhysicalOptimizerRule, WorkerQueryContext}; /// /// struct CustomChannelResolver; /// @@ -266,12 +267,13 @@ pub trait DistributedExt: Sized { /// .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) /// .build(); /// - /// // This function can be provided to an ArrowFlightEndpoint so that, upon receiving a distributed + /// // This function can be provided to a Worker so that, upon receiving a distributed /// // part of a plan, it knows how to resolve gRPC channels from URLs for making network calls to other nodes. - /// async fn build_state(ctx: DistributedSessionBuilderContext) -> Result { - /// Ok(SessionStateBuilder::new() - /// // If you have a custom channel resolver, it should also be passed in the - /// // ArrowFlightEndpoint session builder. + /// async fn build_state(ctx: WorkerQueryContext) -> Result { + /// // If you have a custom channel resolver, it should also be passed in the + /// // Worker session builder. + /// Ok(ctx + /// .builder /// .with_distributed_channel_resolver(CustomChannelResolver) /// .build()) /// } @@ -287,12 +289,12 @@ pub trait DistributedExt: Sized { resolver: T, ); - /// Adds a distributed task count estimator. Estimators are executed on leaf nodes - /// sequentially until one returns an estimation on the amount of tasks that should be - /// used for the stage containing the leaf node. + /// Adds a distributed task count estimator. [TaskEstimator]s are executed on each node + /// sequentially until one returns an estimation on the number of tasks that should be + /// used for the stage containing that node. /// - /// The first one that returns something for a leaf node is the one that decides how many - /// tasks are used. + /// Many nodes might decide to provide an estimation, so a reconciliation between all of them + /// is performed internally during planning. /// /// ```text /// ┌───────────────────────┐ @@ -311,8 +313,8 @@ pub trait DistributedExt: Sized { /// ┌───────────────────────┐ │ /// │ │ FilterExec │ /// └───────────────────────┘ │ - /// │ ┌───────────────────────┐ TaskEstimator estimates tasks in - /// │ SomeExec │◀───┼── stages containing leaf nodes + /// │ ┌───────────────────────┐ a TaskEstimator estimates the amount of tasks + /// │ SomeExec │◀───┼── based on how much data will be pulled. /// │ └───────────────────────┘ /// ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘ /// ``` diff --git a/src/execution_plans/network_coalesce.rs b/src/execution_plans/network_coalesce.rs index 1ea7307c..43200876 100644 --- a/src/execution_plans/network_coalesce.rs +++ b/src/execution_plans/network_coalesce.rs @@ -75,9 +75,9 @@ pub struct NetworkCoalesceExec { /// Metrics are populated in this map via [NetworkCoalesceExec::execute]. /// /// An instance may receive metrics for 0 to N child tasks, where N is the number of tasks in - /// the stage it is reading from. This is because, by convention, the ArrowFlightEndpoint - /// sends metrics for a task to the last NetworkCoalesceExec to read from it, which may or may - /// not be this instance. + /// the stage it is reading from. This is because, by convention, the Worker sends metrics for + /// a task to the last NetworkCoalesceExec to read from it, which may or may not be this + /// instance. pub(crate) metrics_collection: Arc>>, } diff --git a/src/execution_plans/network_shuffle.rs b/src/execution_plans/network_shuffle.rs index 44806f22..6c7a81ea 100644 --- a/src/execution_plans/network_shuffle.rs +++ b/src/execution_plans/network_shuffle.rs @@ -126,7 +126,7 @@ pub struct NetworkShuffleExec { /// Metrics are populated in this map via [NetworkShuffleExec::execute]. /// /// An instance may receive metrics for 0 to N child tasks, where N is the number of tasks in - /// the stage it is reading from. This is because, by convention, the ArrowFlightEndpoint + /// the stage it is reading from. This is because, by convention, the Worker /// sends metrics for a task to the last NetworkShuffleExec to read from it, which may or may /// not be this instance. pub(crate) metrics_collection: Arc>>, diff --git a/src/flight_service/do_get.rs b/src/flight_service/do_get.rs index 6ab6ba06..9921722f 100644 --- a/src/flight_service/do_get.rs +++ b/src/flight_service/do_get.rs @@ -2,8 +2,8 @@ use crate::common::map_last_stream; use crate::config_extension_ext::{ ContextGrpcMetadata, set_distributed_option_extension_from_headers, }; -use crate::flight_service::service::ArrowFlightEndpoint; -use crate::flight_service::session_builder::DistributedSessionBuilderContext; +use crate::flight_service::session_builder::WorkerQueryContext; +use crate::flight_service::worker::Worker; use crate::metrics::TaskMetricsCollector; use crate::metrics::proto::df_metrics_set_to_proto; use crate::protobuf::{ @@ -67,11 +67,11 @@ pub struct TaskData { num_partitions_remaining: Arc, } -impl ArrowFlightEndpoint { +impl Worker { pub(super) async fn get( &self, request: Request, - ) -> Result::DoGetStream>, Status> { + ) -> Result::DoGetStream>, Status> { let (metadata, _ext, body) = request.into_parts(); let doget = DoGet::decode(body.ticket).map_err(|err| { Status::invalid_argument(format!("Cannot decode DoGet message: {err}")) @@ -80,7 +80,7 @@ impl ArrowFlightEndpoint { let headers = metadata.into_headers(); let mut session_state = self .session_builder - .build_session_state(DistributedSessionBuilderContext { + .build_session_state(WorkerQueryContext { builder: SessionStateBuilder::new() .with_default_features() .with_runtime_env(Arc::clone(&self.runtime)), @@ -294,7 +294,7 @@ mod tests { #[tokio::test] async fn test_task_data_partition_counting() { - let mut endpoint = ArrowFlightEndpoint::default(); + let mut endpoint = Worker::default(); let plans_received = Arc::new(AtomicUsize::default()); { let plans_received = Arc::clone(&plans_received); diff --git a/src/flight_service/mod.rs b/src/flight_service/mod.rs index 96ea6090..5c31b1b3 100644 --- a/src/flight_service/mod.rs +++ b/src/flight_service/mod.rs @@ -1,10 +1,10 @@ mod do_get; -mod service; mod session_builder; +mod worker; pub(crate) use do_get::DoGet; -pub use service::ArrowFlightEndpoint; pub use session_builder::{ - DefaultSessionBuilder, DistributedSessionBuilder, DistributedSessionBuilderContext, - MappedDistributedSessionBuilder, MappedDistributedSessionBuilderExt, + DefaultSessionBuilder, MappedWorkerSessionBuilder, MappedWorkerSessionBuilderExt, + WorkerQueryContext, WorkerSessionBuilder, }; +pub use worker::Worker; diff --git a/src/flight_service/session_builder.rs b/src/flight_service/session_builder.rs index d0afdbf8..c0f0bf41 100644 --- a/src/flight_service/session_builder.rs +++ b/src/flight_service/session_builder.rs @@ -5,15 +5,14 @@ use http::HeaderMap; use std::sync::Arc; #[derive(Debug, Default)] -pub struct DistributedSessionBuilderContext { +pub struct WorkerQueryContext { pub builder: SessionStateBuilder, pub headers: HeaderMap, } -/// Trait called by the Arrow Flight endpoint that handles distributed parts of a DataFusion -/// plan for building a DataFusion's [SessionState]. +/// builds a DataFusion's [SessionState] in each query issued to a worker. #[async_trait] -pub trait DistributedSessionBuilder { +pub trait WorkerSessionBuilder { /// Builds a custom [SessionState] scoped to a single ArrowFlight gRPC call, allowing the /// users to provide a customized DataFusion session with things like custom extension codecs, /// custom physical optimization rules, UDFs, UDAFs, config extensions, etc... @@ -27,7 +26,7 @@ pub trait DistributedSessionBuilder { /// # use datafusion::execution::{FunctionRegistry, SessionState, SessionStateBuilder, TaskContext}; /// # use datafusion::physical_plan::ExecutionPlan; /// # use datafusion_proto::physical_plan::PhysicalExtensionCodec; - /// # use datafusion_distributed::{DistributedExt, DistributedSessionBuilder, DistributedSessionBuilderContext}; + /// # use datafusion_distributed::{DistributedExt, WorkerSessionBuilder, WorkerQueryContext}; /// /// #[derive(Debug)] /// struct CustomExecCodec; @@ -46,8 +45,8 @@ pub trait DistributedSessionBuilder { /// struct CustomSessionBuilder; /// /// #[async_trait] - /// impl DistributedSessionBuilder for CustomSessionBuilder { - /// async fn build_session_state(&self, ctx: DistributedSessionBuilderContext) -> Result { + /// impl WorkerSessionBuilder for CustomSessionBuilder { + /// async fn build_session_state(&self, ctx: WorkerQueryContext) -> Result { /// Ok(ctx /// .builder /// .with_distributed_user_codec(CustomExecCodec) @@ -58,52 +57,52 @@ pub trait DistributedSessionBuilder { /// ``` async fn build_session_state( &self, - ctx: DistributedSessionBuilderContext, + ctx: WorkerQueryContext, ) -> Result; } -/// Noop implementation of the [DistributedSessionBuilder]. Used by default if no [DistributedSessionBuilder] is provided -/// while building the Arrow Flight endpoint. +/// Noop implementation of the [WorkerSessionBuilder]. Used by default if no [WorkerSessionBuilder] +/// is provided while building the Worker. #[derive(Debug, Clone)] pub struct DefaultSessionBuilder; #[async_trait] -impl DistributedSessionBuilder for DefaultSessionBuilder { +impl WorkerSessionBuilder for DefaultSessionBuilder { async fn build_session_state( &self, - ctx: DistributedSessionBuilderContext, + ctx: WorkerQueryContext, ) -> Result { Ok(ctx.builder.build()) } } -/// Implementation of [DistributedSessionBuilder] for any async function that returns a [Result] +/// Implementation of [WorkerSessionBuilder] for any async function that returns a [Result] #[async_trait] -impl DistributedSessionBuilder for F +impl WorkerSessionBuilder for F where - F: Fn(DistributedSessionBuilderContext) -> Fut + Send + Sync + 'static, + F: Fn(WorkerQueryContext) -> Fut + Send + Sync + 'static, Fut: std::future::Future> + Send + 'static, { async fn build_session_state( &self, - ctx: DistributedSessionBuilderContext, + ctx: WorkerQueryContext, ) -> Result { self(ctx).await } } -pub trait MappedDistributedSessionBuilderExt { - /// Maps an existing [DistributedSessionBuilder] allowing to add further extensions +pub trait MappedWorkerSessionBuilderExt { + /// Maps an existing [WorkerSessionBuilder] allowing to add further extensions /// to its already built [SessionStateBuilder]. /// - /// Useful if there's already a [DistributedSessionBuilder] that needs to be extended + /// Useful if there's already a [WorkerSessionBuilder] that needs to be extended /// with further capabilities. /// /// Example: /// /// ```rust /// # use datafusion::execution::SessionStateBuilder; - /// # use datafusion_distributed::{DefaultSessionBuilder, MappedDistributedSessionBuilderExt}; + /// # use datafusion_distributed::{DefaultSessionBuilder, MappedWorkerSessionBuilderExt}; /// /// let session_builder = DefaultSessionBuilder /// .map(|b: SessionStateBuilder| { @@ -111,30 +110,30 @@ pub trait MappedDistributedSessionBuilderExt { /// Ok(b.build()) /// }); /// ``` - fn map(self, f: F) -> MappedDistributedSessionBuilder + fn map(self, f: F) -> MappedWorkerSessionBuilder where Self: Sized, F: Fn(SessionStateBuilder) -> Result; } -impl MappedDistributedSessionBuilderExt for T { - fn map(self, f: F) -> MappedDistributedSessionBuilder +impl MappedWorkerSessionBuilderExt for T { + fn map(self, f: F) -> MappedWorkerSessionBuilder where Self: Sized, { - MappedDistributedSessionBuilder { + MappedWorkerSessionBuilder { inner: self, f: Arc::new(f), } } } -pub struct MappedDistributedSessionBuilder { +pub struct MappedWorkerSessionBuilder { inner: T, f: Arc, } -impl Clone for MappedDistributedSessionBuilder { +impl Clone for MappedWorkerSessionBuilder { fn clone(&self) -> Self { Self { inner: self.inner.clone(), @@ -144,14 +143,14 @@ impl Clone for MappedDistributedSessionBuilder { } #[async_trait] -impl DistributedSessionBuilder for MappedDistributedSessionBuilder +impl WorkerSessionBuilder for MappedWorkerSessionBuilder where - T: DistributedSessionBuilder + Send + Sync + 'static, + T: WorkerSessionBuilder + Send + Sync + 'static, F: Fn(SessionStateBuilder) -> Result + Send + Sync, { async fn build_session_state( &self, - ctx: DistributedSessionBuilderContext, + ctx: WorkerQueryContext, ) -> Result { let state = self.inner.build_session_state(ctx).await?; let builder = SessionStateBuilder::new_from_existing(state); diff --git a/src/flight_service/service.rs b/src/flight_service/worker.rs similarity index 85% rename from src/flight_service/service.rs rename to src/flight_service/worker.rs index da631274..ecd7f8b1 100644 --- a/src/flight_service/service.rs +++ b/src/flight_service/worker.rs @@ -1,6 +1,6 @@ use crate::DefaultSessionBuilder; use crate::common::ttl_map::{TTLMap, TTLMapConfig}; -use crate::flight_service::DistributedSessionBuilder; +use crate::flight_service::WorkerSessionBuilder; use crate::flight_service::do_get::TaskData; use crate::protobuf::StageKey; use arrow_flight::flight_service_server::{FlightService, FlightServiceServer}; @@ -18,20 +18,20 @@ use tonic::{Request, Response, Status, Streaming}; #[allow(clippy::type_complexity)] #[derive(Default)] -pub(super) struct ArrowFlightEndpointHooks { +pub(super) struct WorkerHooks { pub(super) on_plan: Vec) -> Arc + Sync + Send>>, } -pub struct ArrowFlightEndpoint { +pub struct Worker { pub(super) runtime: Arc, pub(super) task_data_entries: Arc>>>, - pub(super) session_builder: Arc, - pub(super) hooks: ArrowFlightEndpointHooks, + pub(super) session_builder: Arc, + pub(super) hooks: WorkerHooks, pub(super) max_message_size: Option, } -impl Default for ArrowFlightEndpoint { +impl Default for Worker { fn default() -> Self { let ttl_map = TTLMap::try_new(TTLMapConfig::default()) .expect("Instantiating a TTLMap with default params should never fail"); @@ -39,17 +39,17 @@ impl Default for ArrowFlightEndpoint { runtime: Arc::new(RuntimeEnv::default()), task_data_entries: Arc::new(ttl_map), session_builder: Arc::new(DefaultSessionBuilder), - hooks: ArrowFlightEndpointHooks::default(), + hooks: WorkerHooks::default(), max_message_size: Some(usize::MAX), } } } -impl ArrowFlightEndpoint { - /// Builds an [ArrowFlightEndpoint] with a custom [DistributedSessionBuilder]. Use this +impl Worker { + /// Builds a [Worker] with a custom [WorkerSessionBuilder]. Use this /// method whenever you need to add custom stuff to the `SessionContext` that executes the query. pub fn from_session_builder( - session_builder: impl DistributedSessionBuilder + Send + Sync + 'static, + session_builder: impl WorkerSessionBuilder + Send + Sync + 'static, ) -> Self { Self { session_builder: Arc::new(session_builder), @@ -84,7 +84,7 @@ impl ArrowFlightEndpoint { self } - /// Converts this endpoint into a [`FlightServiceServer`] with high default message size limits. + /// Converts this [Worker] into a [`FlightServiceServer`] with high default message size limits. /// /// This is a convenience method that wraps the endpoint in a [`FlightServiceServer`] and /// configures it with `max_decoding_message_size(usize::MAX)` and @@ -95,11 +95,21 @@ impl ArrowFlightEndpoint { /// /// # Example /// - /// ```rust,ignore - /// let endpoint = ArrowFlightEndpoint::try_new(session_builder)?; - /// let server = endpoint.into_flight_server(); - /// // Can chain additional tonic methods if needed - /// // let server = server.some_other_tonic_method(...); + /// ``` + /// # use datafusion_distributed::Worker; + /// # use tonic::transport::Server; + /// # use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + /// # async fn f() { + /// + /// let worker = Worker::default(); + /// let server = worker.into_flight_server(); + /// + /// Server::builder() + /// .add_service(Worker::default().into_flight_server()) + /// .serve(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080)) + /// .await; + /// + /// # } /// ``` pub fn into_flight_server(self) -> FlightServiceServer { FlightServiceServer::new(self) @@ -109,7 +119,7 @@ impl ArrowFlightEndpoint { } #[async_trait] -impl FlightService for ArrowFlightEndpoint { +impl FlightService for Worker { type HandshakeStream = BoxStream<'static, Result>; async fn handshake( diff --git a/src/lib.rs b/src/lib.rs index cf6a7c76..50e1ef65 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,9 +23,8 @@ pub use execution_plans::{ DistributedExec, NetworkCoalesceExec, NetworkShuffleExec, PartitionIsolatorExec, }; pub use flight_service::{ - ArrowFlightEndpoint, DefaultSessionBuilder, DistributedSessionBuilder, - DistributedSessionBuilderContext, MappedDistributedSessionBuilder, - MappedDistributedSessionBuilderExt, + DefaultSessionBuilder, MappedWorkerSessionBuilder, MappedWorkerSessionBuilderExt, Worker, + WorkerQueryContext, WorkerSessionBuilder, }; pub use metrics::rewrite_distributed_plan_with_metrics; pub use networking::{ diff --git a/src/stage.rs b/src/stage.rs index 14f188a7..5e80c219 100644 --- a/src/stage.rs +++ b/src/stage.rs @@ -16,16 +16,10 @@ use uuid::Uuid; /// It implements [`ExecutionPlan`] and can be executed to produce a /// stream of record batches. /// -/// An ExecutionTask is a finer grained unit of work compared to an ExecutionStage. -/// One ExecutionStage will create one or more ExecutionTasks -/// -/// When an [`StageExec`] is execute()'d if will execute its plan and return a stream -/// of record batches. -/// /// If the stage has input stages, then it those input stages will be executed on remote resources /// and will be provided the remainder of the stage tree. /// -/// For example if our stage tree looks like this: +/// For example, if our stage tree looks like this: /// /// ```text /// ┌─────────┐ @@ -44,13 +38,13 @@ use uuid::Uuid; /// /// ``` /// -/// Then executing Stage 1 will run its plan locally. Stage 1 has two inputs, Stage 2 and Stage 3. We -/// know these will execute on remote resources. As such the plan for Stage 1 must contain an -/// [`NetworkShuffleExec`] node that will read the results of Stage 2 and Stage 3 and coalese the +/// Then executing Stage 1 will run its plan locally. Stage 1 has two inputs, Stage 2 and Stage 3. We +/// know these will execute on remote resources. As such, the plan for Stage 1 must contain a +/// [`NetworkShuffleExec`] node that will read the results of Stage 2 and Stage 3 and coalesce the /// results. /// /// When Stage 1's [`NetworkShuffleExec`] node is executed, it makes an ArrowFlightRequest to the -/// host assigned in the Stage. It provides the following Stage tree serialilzed in the body of the +/// host assigned in the Stage. It provides the following Stage tree serialized in the body of the /// Arrow Flight Ticket: /// /// ```text @@ -65,7 +59,7 @@ use uuid::Uuid; /// /// ``` /// -/// The receiving ArrowFlightEndpoint will then execute Stage 2 and will repeat this process. +/// The receiving Worker will then execute Stage 2 and will repeat this process. /// /// When Stage 4 is executed, it has no input tasks, so it is assumed that the plan included in that /// Stage can complete on its own; it's likely holding a leaf node in the overall physical plan and @@ -178,7 +172,7 @@ use datafusion_proto::protobuf::PhysicalPlanNode; use prost::Message; /// Be able to display a nice tree for stages. /// -/// The challenge to doing this at the moment is that `TreeRenderVistor` +/// The challenge to doing this at the moment is that `TreeRenderVisitor` /// in [`datafusion::physical_plan::display`] is not public, and that it also /// is specific to a `ExecutionPlan` trait object, which we don't have. /// @@ -334,7 +328,7 @@ const COLOR_SCHEME: &str = "spectral6"; /// Graphviz dot format. /// You can view them on https://vis-js.com /// -/// Or it is often useful to expertiment with plan output using +/// Or it is often useful to experiment with plan output using /// https://datafusion-fiddle.vercel.app/ pub fn display_plan_graphviz(plan: Arc) -> Result { let mut f = String::new(); diff --git a/src/test_utils/in_memory_channel_resolver.rs b/src/test_utils/in_memory_channel_resolver.rs index 8e61006c..96baef2d 100644 --- a/src/test_utils/in_memory_channel_resolver.rs +++ b/src/test_utils/in_memory_channel_resolver.rs @@ -1,6 +1,6 @@ use crate::{ - ArrowFlightEndpoint, BoxCloneSyncChannel, ChannelResolver, DefaultSessionBuilder, - DistributedExt, DistributedSessionBuilder, MappedDistributedSessionBuilderExt, WorkerResolver, + BoxCloneSyncChannel, ChannelResolver, DefaultSessionBuilder, DistributedExt, + MappedWorkerSessionBuilderExt, Worker, WorkerResolver, WorkerSessionBuilder, create_flight_client, }; use arrow_flight::flight_service_client::FlightServiceClient; @@ -19,11 +19,11 @@ pub struct InMemoryChannelResolver { } impl InMemoryChannelResolver { - /// Build an [InMemoryChannelResolver] with a custom [DistributedSessionBuilder]. + /// Build an [InMemoryChannelResolver] with a custom [WorkerSessionBuilder]. /// This allows you to inject your own DataFusion extensions in the in-memory worker /// spawned by this method. pub fn from_session_builder( - builder: impl DistributedSessionBuilder + Send + Sync + 'static, + builder: impl WorkerSessionBuilder + Send + Sync + 'static, ) -> Self { let (client, server) = tokio::io::duplex(1024 * 1024); @@ -42,7 +42,7 @@ impl InMemoryChannelResolver { }; let this_clone = this.clone(); - let endpoint = ArrowFlightEndpoint::from_session_builder(builder.map(move |builder| { + let endpoint = Worker::from_session_builder(builder.map(move |builder| { let this = this.clone(); Ok(builder.with_distributed_channel_resolver(this).build()) })); diff --git a/src/test_utils/localhost.rs b/src/test_utils/localhost.rs index 131946a7..e6622a06 100644 --- a/src/test_utils/localhost.rs +++ b/src/test_utils/localhost.rs @@ -1,6 +1,6 @@ use crate::{ - ArrowFlightEndpoint, DistributedExt, DistributedPhysicalOptimizerRule, - DistributedSessionBuilder, DistributedSessionBuilderContext, WorkerResolver, + DistributedExt, DistributedPhysicalOptimizerRule, Worker, WorkerQueryContext, WorkerResolver, + WorkerSessionBuilder, }; use async_trait::async_trait; use datafusion::common::DataFusionError; @@ -26,7 +26,7 @@ pub async fn start_localhost_context( session_builder: B, ) -> (SessionContext, JoinSet<()>) where - B: DistributedSessionBuilder + Send + Sync + 'static, + B: WorkerSessionBuilder + Send + Sync + 'static, B: Clone, { let listeners = futures::future::try_join_all( @@ -60,7 +60,7 @@ where let worker_resolver = LocalHostWorkerResolver::new(ports); let mut state = session_builder - .build_session_state(DistributedSessionBuilderContext { + .build_session_state(WorkerQueryContext { builder: SessionStateBuilder::new() .with_default_features() .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) @@ -102,10 +102,10 @@ impl WorkerResolver for LocalHostWorkerResolver { } pub async fn spawn_flight_service( - session_builder: impl DistributedSessionBuilder + Send + Sync + 'static, + session_builder: impl WorkerSessionBuilder + Send + Sync + 'static, incoming: TcpListener, ) -> Result<(), Box> { - let endpoint = ArrowFlightEndpoint::from_session_builder(session_builder); + let endpoint = Worker::from_session_builder(session_builder); let incoming = tokio_stream::wrappers::TcpListenerStream::new(incoming); diff --git a/tests/custom_config_extension.rs b/tests/custom_config_extension.rs index 9d4356c0..58274313 100644 --- a/tests/custom_config_extension.rs +++ b/tests/custom_config_extension.rs @@ -15,7 +15,7 @@ mod tests { }; use datafusion_distributed::test_utils::localhost::start_localhost_context; use datafusion_distributed::test_utils::parquet::register_parquet_tables; - use datafusion_distributed::{DistributedExt, DistributedSessionBuilderContext}; + use datafusion_distributed::{DistributedExt, WorkerQueryContext}; use datafusion_proto::physical_plan::PhysicalExtensionCodec; use futures::TryStreamExt; use prost::Message; @@ -23,9 +23,7 @@ mod tests { use std::fmt::Formatter; use std::sync::Arc; - async fn build_state( - ctx: DistributedSessionBuilderContext, - ) -> Result { + async fn build_state(ctx: WorkerQueryContext) -> Result { Ok(ctx .builder .with_distributed_option_extension_from_headers::(&ctx.headers)? diff --git a/tests/custom_extension_codec.rs b/tests/custom_extension_codec.rs index 3ef49fe0..d1f2a22d 100644 --- a/tests/custom_extension_codec.rs +++ b/tests/custom_extension_codec.rs @@ -12,9 +12,7 @@ mod tests { }; use datafusion_distributed::test_utils::localhost::start_localhost_context; use datafusion_distributed::test_utils::parquet::register_parquet_tables; - use datafusion_distributed::{ - DistributedExt, DistributedSessionBuilderContext, assert_snapshot, - }; + use datafusion_distributed::{DistributedExt, WorkerQueryContext, assert_snapshot}; use datafusion_proto::physical_plan::PhysicalExtensionCodec; use datafusion_proto::protobuf::proto_error; use futures::TryStreamExt; @@ -25,9 +23,7 @@ mod tests { #[tokio::test] async fn custom_extension_codec() -> Result<(), Box> { - async fn build_state( - ctx: DistributedSessionBuilderContext, - ) -> Result { + async fn build_state(ctx: WorkerQueryContext) -> Result { Ok(ctx .builder .with_distributed_user_codec(CustomPassThroughExecCodec) diff --git a/tests/error_propagation.rs b/tests/error_propagation.rs index cf042481..dbb3b7c5 100644 --- a/tests/error_propagation.rs +++ b/tests/error_propagation.rs @@ -12,7 +12,7 @@ mod tests { }; use datafusion_distributed::test_utils::localhost::start_localhost_context; use datafusion_distributed::test_utils::parquet::register_parquet_tables; - use datafusion_distributed::{DistributedExt, DistributedSessionBuilderContext}; + use datafusion_distributed::{DistributedExt, WorkerQueryContext}; use datafusion_proto::physical_plan::PhysicalExtensionCodec; use datafusion_proto::protobuf::proto_error; use futures::{TryStreamExt, stream}; @@ -24,9 +24,7 @@ mod tests { #[tokio::test] async fn test_error_propagation() -> Result<(), Box> { - async fn build_state( - ctx: DistributedSessionBuilderContext, - ) -> Result { + async fn build_state(ctx: WorkerQueryContext) -> Result { Ok(ctx .builder .with_distributed_user_codec(ErrorThrowingExecCodec) diff --git a/tests/introspection.rs b/tests/introspection.rs index 3a7a5a8a..18d186b9 100644 --- a/tests/introspection.rs +++ b/tests/introspection.rs @@ -4,22 +4,19 @@ mod tests { use datafusion::physical_plan::execute_stream; use datafusion_distributed::test_utils::localhost::start_localhost_context; use datafusion_distributed::test_utils::parquet::register_parquet_tables; - use datafusion_distributed::{ - DistributedSessionBuilderContext, assert_snapshot, display_plan_ascii, - }; + use datafusion_distributed::{WorkerQueryContext, assert_snapshot, display_plan_ascii}; use futures::TryStreamExt; use std::error::Error; #[tokio::test] async fn distributed_show_columns() -> Result<(), Box> { - let (ctx, _guard) = - start_localhost_context(3, |mut ctx: DistributedSessionBuilderContext| async { - let cfg = ctx.builder.config().get_or_insert_default(); - let opts = cfg.options_mut(); - opts.catalog.information_schema = true; - Ok(ctx.builder.build()) - }) - .await; + let (ctx, _guard) = start_localhost_context(3, |mut ctx: WorkerQueryContext| async { + let cfg = ctx.builder.config().get_or_insert_default(); + let opts = cfg.options_mut(); + opts.catalog.information_schema = true; + Ok(ctx.builder.build()) + }) + .await; register_parquet_tables(&ctx).await?; let df = ctx.sql(r#"SHOW COLUMNS from weather"#).await?; diff --git a/tests/stateful_execution_plan.rs b/tests/stateful_execution_plan.rs index e6ec725f..b9bf8fd9 100644 --- a/tests/stateful_execution_plan.rs +++ b/tests/stateful_execution_plan.rs @@ -14,7 +14,7 @@ mod tests { use datafusion_distributed::test_utils::localhost::start_localhost_context; use datafusion_distributed::test_utils::parquet::register_parquet_tables; use datafusion_distributed::{ - DistributedExt, DistributedSessionBuilderContext, assert_snapshot, display_plan_ascii, + DistributedExt, WorkerQueryContext, assert_snapshot, display_plan_ascii, }; use datafusion_proto::physical_plan::PhysicalExtensionCodec; use datafusion_proto::protobuf::proto_error; @@ -27,17 +27,15 @@ mod tests { use tokio_stream::StreamExt; use tokio_stream::wrappers::ReceiverStream; - // This test proves that execution nodes do not get early dropped in the ArrowFlightEndpoint - // when all the partitions start being consumed. + // This test proves that execution nodes do not get early dropped in the Worker when all the + // partitions start being consumed. // // It uses a StatefulPassThroughExec custom node whose execution depends on it not being dropped. // The node spawns a background task that collects data from its child DataSourceExec. - // If ArrowFlightEndpoint drops the node before the stream ends, this test will fail. + // If the Worker drops the node before the stream ends, this test will fail. #[tokio::test] async fn stateful_execution_plan() -> Result<(), Box> { - async fn build_state( - ctx: DistributedSessionBuilderContext, - ) -> Result { + async fn build_state(ctx: WorkerQueryContext) -> Result { Ok(ctx .builder .with_distributed_user_codec(PassThroughExecCodec) diff --git a/tests/udfs.rs b/tests/udfs.rs index e99c5bef..ae9f96f4 100644 --- a/tests/udfs.rs +++ b/tests/udfs.rs @@ -16,8 +16,8 @@ mod tests { use datafusion::physical_plan::{ExecutionPlan, execute_stream}; use datafusion_distributed::test_utils::localhost::start_localhost_context; use datafusion_distributed::{ - DistributedExt, DistributedPhysicalOptimizerRule, DistributedSessionBuilderContext, - assert_snapshot, display_plan_ascii, + DistributedExt, DistributedPhysicalOptimizerRule, WorkerQueryContext, assert_snapshot, + display_plan_ascii, }; use futures::TryStreamExt; use std::any::Any; @@ -26,9 +26,7 @@ mod tests { #[tokio::test] async fn test_udf_in_partitioning_field() -> Result<(), Box> { - async fn build_state( - ctx: DistributedSessionBuilderContext, - ) -> Result { + async fn build_state(ctx: WorkerQueryContext) -> Result { Ok(ctx .builder .with_scalar_functions(vec![udf()]) From 48910a62fc25b4fa1bb217c8737ef22233c475bf Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Fri, 2 Jan 2026 08:13:56 +0100 Subject: [PATCH 11/27] Improve default task estimator (#275) --- src/distributed_planner/task_estimator.rs | 18 +- tests/clickbench_plans_test.rs | 545 +++++---- tests/tpcds_plans_test.rs | 1298 +++++++++++---------- 3 files changed, 952 insertions(+), 909 deletions(-) diff --git a/src/distributed_planner/task_estimator.rs b/src/distributed_planner/task_estimator.rs index 2bf74ab0..d8d5c283 100644 --- a/src/distributed_planner/task_estimator.rs +++ b/src/distributed_planner/task_estimator.rs @@ -6,7 +6,6 @@ use datafusion::datasource::physical_plan::FileScanConfig; use datafusion::physical_plan::ExecutionPlan; use datafusion::prelude::SessionConfig; use delegate::delegate; -use std::collections::HashSet; use std::fmt::Debug; use std::sync::Arc; @@ -205,22 +204,15 @@ impl TaskEstimator for FileScanConfigTaskEstimator { let d_cfg = cfg.extensions.get::()?; - // Count how many distinct files we have in the FileScanConfig. Each file in each - // file group is a PartitionedFile rather than a full file, so it's possible that - // many entries refer to different chunks of the same physical file. By keeping a - // HashSet of the different locations of the PartitionedFiles we count how many actual - // different files we have. - let mut distinct_files = HashSet::new(); + // Count how many partitioned files we have in the FileScanConfig. + let mut partitioned_files = 0; for file_group in &file_scan.file_groups { - for file in file_group.iter() { - distinct_files.insert(file.object_meta.location.clone()); - } + partitioned_files += file_group.len(); } - let distinct_files = distinct_files.len(); // Based on the user-provided files_per_task configuration, do the math to calculate // how many tasks should be used, without surpassing the number of available workers. - let task_count = distinct_files.div_ceil(d_cfg.files_per_task); + let task_count = partitioned_files.div_ceil(d_cfg.files_per_task); Some(TaskEstimation { task_count: TaskCountAnnotation::Desired(task_count), @@ -238,7 +230,7 @@ impl TaskEstimator for FileScanConfigTaskEstimator { } // Based on the task count, attempt to scale up the partitions in the DataSourceExec by // repartitioning it. This will result in a DataSourceExec with potentially a lot of - // partitions, but as we are going wrap it with PartitionIsolatorExec that's fine. + // partitions, but as we are going to wrap it with PartitionIsolatorExec, that's fine. let dse: &DataSourceExec = plan.as_any().downcast_ref()?; let file_scan: &FileScanConfig = dse.data_source().as_any().downcast_ref()?; diff --git a/tests/clickbench_plans_test.rs b/tests/clickbench_plans_test.rs index 907c51d2..e9e07432 100644 --- a/tests/clickbench_plans_test.rs +++ b/tests/clickbench_plans_test.rs @@ -31,14 +31,14 @@ mod tests { │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] │ ProjectionExec: expr=[] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: AdvEngineID@0 != 0 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[AdvEngineID], file_type=parquet, predicate=AdvEngineID@0 != 0, pruning_predicate=AdvEngineID_null_count@2 != row_count@3 AND (AdvEngineID_min@0 != 0 OR 0 != AdvEngineID_max@1), required_guarantees=[AdvEngineID not in (0)] └────────────────────────────────────────────────── "); @@ -53,11 +53,11 @@ mod tests { │ ProjectionExec: expr=[sum(hits.AdvEngineID)@0 as sum(hits.AdvEngineID), count(Int64(1))@1 as count(*), avg(hits.ResolutionWidth)@2 as avg(hits.ResolutionWidth)] │ AggregateExec: mode=Final, gby=[], aggr=[sum(hits.AdvEngineID), count(Int64(1)), avg(hits.ResolutionWidth)] │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[sum(hits.AdvEngineID), count(Int64(1)), avg(hits.ResolutionWidth)] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[ResolutionWidth, AdvEngineID], file_type=parquet └────────────────────────────────────────────────── "); @@ -96,13 +96,13 @@ mod tests { ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ AggregateExec: mode=Partial, gby=[], aggr=[count(alias1)] │ AggregateExec: mode=FinalPartitioned, gby=[alias1@0 as alias1], aggr=[] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([alias1@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([alias1@0], 6), input_partitions=2 │ AggregateExec: mode=Partial, gby=[UserID@0 as alias1], aggr=[] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID], file_type=parquet └────────────────────────────────────────────────── "); @@ -122,13 +122,13 @@ mod tests { ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ AggregateExec: mode=Partial, gby=[], aggr=[count(alias1)] │ AggregateExec: mode=FinalPartitioned, gby=[alias1@0 as alias1], aggr=[] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([alias1@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([alias1@0], 6), input_partitions=2 │ AggregateExec: mode=Partial, gby=[SearchPhrase@0 as alias1], aggr=[] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchPhrase], file_type=parquet └────────────────────────────────────────────────── "); @@ -150,20 +150,23 @@ mod tests { ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[AdvEngineID@0 as AdvEngineID, count(*)@1 as count(*)] │ SortPreservingMergeExec: [count(Int64(1))@2 DESC] - │ SortExec: expr=[count(*)@1 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[AdvEngineID@0 as AdvEngineID, count(Int64(1))@1 as count(*), count(Int64(1))@1 as count(Int64(1))] - │ AggregateExec: mode=FinalPartitioned, gby=[AdvEngineID@0 as AdvEngineID], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([AdvEngineID@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[AdvEngineID@0 as AdvEngineID], aggr=[count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: AdvEngineID@0 != 0 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] - │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[AdvEngineID], file_type=parquet, predicate=AdvEngineID@0 != 0, pruning_predicate=AdvEngineID_null_count@2 != row_count@3 AND (AdvEngineID_min@0 != 0 OR 0 != AdvEngineID_max@1), required_guarantees=[AdvEngineID not in (0)] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: expr=[count(*)@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[AdvEngineID@0 as AdvEngineID, count(Int64(1))@1 as count(*), count(Int64(1))@1 as count(Int64(1))] + │ AggregateExec: mode=FinalPartitioned, gby=[AdvEngineID@0 as AdvEngineID], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([AdvEngineID@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[AdvEngineID@0 as AdvEngineID], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: AdvEngineID@0 != 0 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[AdvEngineID], file_type=parquet, predicate=AdvEngineID@0 != 0, pruning_predicate=AdvEngineID_null_count@2 != row_count@3 AND (AdvEngineID_min@0 != 0 OR 0 != AdvEngineID_max@1), required_guarantees=[AdvEngineID not in (0)] + └────────────────────────────────────────────────── "); Ok(()) } @@ -184,13 +187,13 @@ mod tests { │ RepartitionExec: partitioning=Hash([RegionID@0], 3), input_partitions=3 │ AggregateExec: mode=Partial, gby=[RegionID@0 as RegionID], aggr=[count(alias1)] │ AggregateExec: mode=FinalPartitioned, gby=[RegionID@0 as RegionID, alias1@1 as alias1], aggr=[] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([RegionID@0, alias1@1], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([RegionID@0, alias1@1], 6), input_partitions=2 │ AggregateExec: mode=Partial, gby=[RegionID@0 as RegionID, UserID@1 as alias1], aggr=[] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[RegionID, UserID], file_type=parquet └────────────────────────────────────────────────── "); @@ -209,13 +212,13 @@ mod tests { │ SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true] │ ProjectionExec: expr=[RegionID@0 as RegionID, sum(hits.AdvEngineID)@1 as sum(hits.AdvEngineID), count(Int64(1))@2 as c, avg(hits.ResolutionWidth)@3 as avg(hits.ResolutionWidth), count(DISTINCT hits.UserID)@4 as count(DISTINCT hits.UserID)] │ AggregateExec: mode=FinalPartitioned, gby=[RegionID@0 as RegionID], aggr=[sum(hits.AdvEngineID), count(Int64(1)), avg(hits.ResolutionWidth), count(DISTINCT hits.UserID)] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([RegionID@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([RegionID@0], 6), input_partitions=2 │ AggregateExec: mode=Partial, gby=[RegionID@0 as RegionID], aggr=[sum(hits.AdvEngineID), count(Int64(1)), avg(hits.ResolutionWidth), count(DISTINCT hits.UserID)] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[RegionID, UserID, ResolutionWidth, AdvEngineID], file_type=parquet └────────────────────────────────────────────────── "); @@ -231,21 +234,24 @@ mod tests { │ SortExec: TopK(fetch=10), expr=[u@1 DESC], preserve_partitioning=[true] │ ProjectionExec: expr=[MobilePhoneModel@0 as MobilePhoneModel, count(alias1)@1 as u] │ AggregateExec: mode=FinalPartitioned, gby=[MobilePhoneModel@0 as MobilePhoneModel], aggr=[count(alias1)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([MobilePhoneModel@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[MobilePhoneModel@0 as MobilePhoneModel], aggr=[count(alias1)] - │ AggregateExec: mode=FinalPartitioned, gby=[MobilePhoneModel@0 as MobilePhoneModel, alias1@1 as alias1], aggr=[] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([MobilePhoneModel@0, alias1@1], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[MobilePhoneModel@1 as MobilePhoneModel, UserID@0 as alias1], aggr=[] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: MobilePhoneModel@1 != - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] - │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID, MobilePhoneModel], file_type=parquet, predicate=MobilePhoneModel@1 != , pruning_predicate=MobilePhoneModel_null_count@2 != row_count@3 AND (MobilePhoneModel_min@0 != OR != MobilePhoneModel_max@1), required_guarantees=[MobilePhoneModel not in ()] + │ RepartitionExec: partitioning=Hash([MobilePhoneModel@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[MobilePhoneModel@0 as MobilePhoneModel], aggr=[count(alias1)] + │ AggregateExec: mode=FinalPartitioned, gby=[MobilePhoneModel@0 as MobilePhoneModel, alias1@1 as alias1], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([MobilePhoneModel@0, alias1@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[MobilePhoneModel@1 as MobilePhoneModel, UserID@0 as alias1], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: MobilePhoneModel@1 != + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID, MobilePhoneModel], file_type=parquet, predicate=MobilePhoneModel@1 != , pruning_predicate=MobilePhoneModel_null_count@2 != row_count@3 AND (MobilePhoneModel_min@0 != OR != MobilePhoneModel_max@1), required_guarantees=[MobilePhoneModel not in ()] + └────────────────────────────────────────────────── "); Ok(()) } @@ -259,21 +265,24 @@ mod tests { │ SortExec: TopK(fetch=10), expr=[u@2 DESC], preserve_partitioning=[true] │ ProjectionExec: expr=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel, count(alias1)@2 as u] │ AggregateExec: mode=FinalPartitioned, gby=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel], aggr=[count(alias1)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([MobilePhone@0, MobilePhoneModel@1], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel], aggr=[count(alias1)] - │ AggregateExec: mode=FinalPartitioned, gby=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel, alias1@2 as alias1], aggr=[] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([MobilePhone@0, MobilePhoneModel@1, alias1@2], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[MobilePhone@1 as MobilePhone, MobilePhoneModel@2 as MobilePhoneModel, UserID@0 as alias1], aggr=[] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: MobilePhoneModel@2 != - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] - │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID, MobilePhone, MobilePhoneModel], file_type=parquet, predicate=MobilePhoneModel@2 != , pruning_predicate=MobilePhoneModel_null_count@2 != row_count@3 AND (MobilePhoneModel_min@0 != OR != MobilePhoneModel_max@1), required_guarantees=[MobilePhoneModel not in ()] + │ RepartitionExec: partitioning=Hash([MobilePhone@0, MobilePhoneModel@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel], aggr=[count(alias1)] + │ AggregateExec: mode=FinalPartitioned, gby=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel, alias1@2 as alias1], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([MobilePhone@0, MobilePhoneModel@1, alias1@2], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[MobilePhone@1 as MobilePhone, MobilePhoneModel@2 as MobilePhoneModel, UserID@0 as alias1], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: MobilePhoneModel@2 != + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID, MobilePhone, MobilePhoneModel], file_type=parquet, predicate=MobilePhoneModel@2 != , pruning_predicate=MobilePhoneModel_null_count@2 != row_count@3 AND (MobilePhoneModel_min@0 != OR != MobilePhoneModel_max@1), required_guarantees=[MobilePhoneModel not in ()] + └────────────────────────────────────────────────── "); Ok(()) } @@ -284,20 +293,23 @@ mod tests { assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [c@1 DESC], fetch=10 - │ SortExec: TopK(fetch=10), expr=[c@1 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase, count(Int64(1))@1 as c] - │ AggregateExec: mode=FinalPartitioned, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([SearchPhrase@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: SearchPhrase@0 != - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] - │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchPhrase], file_type=parquet, predicate=SearchPhrase@0 != , pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[c@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase, count(Int64(1))@1 as c] + │ AggregateExec: mode=FinalPartitioned, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([SearchPhrase@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: SearchPhrase@0 != + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchPhrase], file_type=parquet, predicate=SearchPhrase@0 != , pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + └────────────────────────────────────────────────── "); Ok(()) } @@ -311,21 +323,24 @@ mod tests { │ SortExec: TopK(fetch=10), expr=[u@1 DESC], preserve_partitioning=[true] │ ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase, count(alias1)@1 as u] │ AggregateExec: mode=FinalPartitioned, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(alias1)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([SearchPhrase@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(alias1)] - │ AggregateExec: mode=FinalPartitioned, gby=[SearchPhrase@0 as SearchPhrase, alias1@1 as alias1], aggr=[] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([SearchPhrase@0, alias1@1], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[SearchPhrase@1 as SearchPhrase, UserID@0 as alias1], aggr=[] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: SearchPhrase@1 != - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] - │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID, SearchPhrase], file_type=parquet, predicate=SearchPhrase@1 != , pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + │ RepartitionExec: partitioning=Hash([SearchPhrase@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(alias1)] + │ AggregateExec: mode=FinalPartitioned, gby=[SearchPhrase@0 as SearchPhrase, alias1@1 as alias1], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([SearchPhrase@0, alias1@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[SearchPhrase@1 as SearchPhrase, UserID@0 as alias1], aggr=[] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: SearchPhrase@1 != + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID, SearchPhrase], file_type=parquet, predicate=SearchPhrase@1 != , pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + └────────────────────────────────────────────────── "); Ok(()) } @@ -336,20 +351,23 @@ mod tests { assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [c@2 DESC], fetch=10 - │ SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[SearchEngineID@0 as SearchEngineID, SearchPhrase@1 as SearchPhrase, count(Int64(1))@2 as c] - │ AggregateExec: mode=FinalPartitioned, gby=[SearchEngineID@0 as SearchEngineID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([SearchEngineID@0, SearchPhrase@1], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[SearchEngineID@0 as SearchEngineID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: SearchPhrase@1 != - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] - │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchEngineID, SearchPhrase], file_type=parquet, predicate=SearchPhrase@1 != , pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[SearchEngineID@0 as SearchEngineID, SearchPhrase@1 as SearchPhrase, count(Int64(1))@2 as c] + │ AggregateExec: mode=FinalPartitioned, gby=[SearchEngineID@0 as SearchEngineID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([SearchEngineID@0, SearchPhrase@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[SearchEngineID@0 as SearchEngineID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: SearchPhrase@1 != + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchEngineID, SearchPhrase], file_type=parquet, predicate=SearchPhrase@1 != , pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + └────────────────────────────────────────────────── "); Ok(()) } @@ -367,13 +385,13 @@ mod tests { │ SortExec: TopK(fetch=10), expr=[count(*)@1 DESC], preserve_partitioning=[true] │ ProjectionExec: expr=[UserID@0 as UserID, count(Int64(1))@1 as count(*), count(Int64(1))@1 as count(Int64(1))] │ AggregateExec: mode=FinalPartitioned, gby=[UserID@0 as UserID], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([UserID@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([UserID@0], 6), input_partitions=2 │ AggregateExec: mode=Partial, gby=[UserID@0 as UserID], aggr=[count(Int64(1))] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID], file_type=parquet └────────────────────────────────────────────────── "); @@ -393,13 +411,13 @@ mod tests { │ SortExec: TopK(fetch=10), expr=[count(*)@2 DESC], preserve_partitioning=[true] │ ProjectionExec: expr=[UserID@0 as UserID, SearchPhrase@1 as SearchPhrase, count(Int64(1))@2 as count(*), count(Int64(1))@2 as count(Int64(1))] │ AggregateExec: mode=FinalPartitioned, gby=[UserID@0 as UserID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([UserID@0, SearchPhrase@1], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([UserID@0, SearchPhrase@1], 6), input_partitions=2 │ AggregateExec: mode=Partial, gby=[UserID@0 as UserID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID, SearchPhrase], file_type=parquet └────────────────────────────────────────────────── "); @@ -427,13 +445,13 @@ mod tests { │ SortExec: TopK(fetch=10), expr=[count(*)@3 DESC], preserve_partitioning=[true] │ ProjectionExec: expr=[UserID@0 as UserID, date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime))@1 as m, SearchPhrase@2 as SearchPhrase, count(Int64(1))@3 as count(*), count(Int64(1))@3 as count(Int64(1))] │ AggregateExec: mode=FinalPartitioned, gby=[UserID@0 as UserID, date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime))@1 as date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime)), SearchPhrase@2 as SearchPhrase], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([UserID@0, date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime))@1, SearchPhrase@2], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([UserID@0, date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime))@1, SearchPhrase@2], 6), input_partitions=2 │ AggregateExec: mode=Partial, gby=[UserID@1 as UserID, date_part(MINUTE, to_timestamp_seconds(EventTime@0)) as date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime)), SearchPhrase@2 as SearchPhrase], aggr=[count(Int64(1))] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventTime, UserID, SearchPhrase], file_type=parquet └────────────────────────────────────────────────── "#); @@ -446,12 +464,12 @@ mod tests { assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: UserID@0 = 435090932899640449 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID], file_type=parquet, predicate=UserID@0 = 435090932899640449, pruning_predicate=UserID_null_count@2 != row_count@3 AND UserID_min@0 <= 435090932899640449 AND 435090932899640449 <= UserID_max@1, required_guarantees=[UserID in (435090932899640449)] └────────────────────────────────────────────────── "); @@ -466,14 +484,14 @@ mod tests { │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] │ ProjectionExec: expr=[] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: CAST(URL@0 AS Utf8View) LIKE %google% - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[URL], file_type=parquet, predicate=CAST(URL@0 AS Utf8View) LIKE %google% └────────────────────────────────────────────────── "); @@ -502,13 +520,13 @@ mod tests { assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [EventTime@4 ASC NULLS LAST], fetch=10 - │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ SortExec: TopK(fetch=10), expr=[EventTime@4 ASC NULLS LAST], preserve_partitioning=[true] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: CAST(URL@13 AS Utf8View) LIKE %google% - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[WatchID, JavaEnable, Title, GoodEvent, EventTime, EventDate, CounterID, ClientIP, RegionID, UserID, CounterClass, OS, UserAgent, URL, Referer, IsRefresh, RefererCategoryID, RefererRegionID, URLCategoryID, URLRegionID, ResolutionWidth, ResolutionHeight, ResolutionDepth, FlashMajor, FlashMinor, FlashMinor2, NetMajor, NetMinor, UserAgentMajor, UserAgentMinor, CookieEnable, JavascriptEnable, IsMobile, MobilePhone, MobilePhoneModel, Params, IPNetworkID, TraficSourceID, SearchEngineID, SearchPhrase, AdvEngineID, IsArtifical, WindowClientWidth, WindowClientHeight, ClientTimeZone, ClientEventTime, SilverlightVersion1, SilverlightVersion2, SilverlightVersion3, SilverlightVersion4, PageCharset, CodeVersion, IsLink, IsDownload, IsNotBounce, FUniqID, OriginalURL, HID, IsOldCounter, IsEvent, IsParameter, DontCountHits, WithHash, HitColor, LocalEventTime, Age, Sex, Income, Interests, Robotness, RemoteIP, WindowName, OpenerName, HistoryLength, BrowserLanguage, BrowserCountry, SocialNetwork, SocialAction, HTTPError, SendTiming, DNSTiming, ConnectTiming, ResponseStartTiming, ResponseEndTiming, FetchTiming, SocialSourceNetworkID, SocialSourcePage, ParamPrice, ParamOrderID, ParamCurrency, ParamCurrencyID, OpenstatServiceName, OpenstatCampaignID, OpenstatAdID, OpenstatSourceID, UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm, FromTag, HasGCLID, RefererHash, URLHash, CLID], file_type=parquet, predicate=CAST(URL@13 AS Utf8View) LIKE %google% AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); @@ -529,13 +547,13 @@ mod tests { assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [SearchPhrase@0 ASC NULLS LAST], fetch=10 - │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ SortExec: TopK(fetch=10), expr=[SearchPhrase@0 ASC NULLS LAST], preserve_partitioning=[true] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: SearchPhrase@0 != - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchPhrase], file_type=parquet, predicate=SearchPhrase@0 != AND DynamicFilter [ empty ], pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] └────────────────────────────────────────────────── "); @@ -549,14 +567,14 @@ mod tests { ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase] │ SortPreservingMergeExec: [EventTime@1 ASC NULLS LAST, SearchPhrase@0 ASC NULLS LAST], fetch=10 - │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ SortExec: TopK(fetch=10), expr=[EventTime@1 ASC NULLS LAST, SearchPhrase@0 ASC NULLS LAST], preserve_partitioning=[true] │ ProjectionExec: expr=[SearchPhrase@1 as SearchPhrase, EventTime@0 as EventTime] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: SearchPhrase@1 != - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventTime, SearchPhrase], file_type=parquet, predicate=SearchPhrase@1 != AND DynamicFilter [ empty ], pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] └────────────────────────────────────────────────── "); @@ -569,22 +587,25 @@ mod tests { assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [l@1 DESC], fetch=25 - │ SortExec: TopK(fetch=25), expr=[l@1 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[CounterID@0 as CounterID, avg(length(hits.URL))@1 as l, count(Int64(1))@2 as c] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: count(Int64(1))@2 > 100000 - │ AggregateExec: mode=FinalPartitioned, gby=[CounterID@0 as CounterID], aggr=[avg(length(hits.URL)), count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CounterID@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[CounterID@0 as CounterID], aggr=[avg(length(hits.URL)), count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: URL@1 != - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] - │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[CounterID, URL], file_type=parquet, predicate=URL@1 != , pruning_predicate=URL_null_count@2 != row_count@3 AND (URL_min@0 != OR != URL_max@1), required_guarantees=[URL not in ()] + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=25), expr=[l@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[CounterID@0 as CounterID, avg(length(hits.URL))@1 as l, count(Int64(1))@2 as c] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: count(Int64(1))@2 > 100000 + │ AggregateExec: mode=FinalPartitioned, gby=[CounterID@0 as CounterID], aggr=[avg(length(hits.URL)), count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([CounterID@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[CounterID@0 as CounterID], aggr=[avg(length(hits.URL)), count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: URL@1 != + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[CounterID, URL], file_type=parquet, predicate=URL@1 != , pruning_predicate=URL_null_count@2 != row_count@3 AND (URL_min@0 != OR != URL_max@1), required_guarantees=[URL not in ()] + └────────────────────────────────────────────────── "); Ok(()) } @@ -595,22 +616,25 @@ mod tests { assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [l@1 DESC], fetch=25 - │ SortExec: TopK(fetch=25), expr=[l@1 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0 as k, avg(length(hits.Referer))@1 as l, count(Int64(1))@2 as c, min(hits.Referer)@3 as min(hits.Referer)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: count(Int64(1))@2 > 100000 - │ AggregateExec: mode=FinalPartitioned, gby=[regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0 as regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))], aggr=[avg(length(hits.Referer)), count(Int64(1)), min(hits.Referer)] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[regexp_replace(CAST(Referer@0 AS LargeUtf8), ^https?://(?:www\.)?([^/]+)/.*$, \1) as regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))], aggr=[avg(length(hits.Referer)), count(Int64(1)), min(hits.Referer)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: Referer@0 != - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] - │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[Referer], file_type=parquet, predicate=Referer@0 != , pruning_predicate=Referer_null_count@2 != row_count@3 AND (Referer_min@0 != OR != Referer_max@1), required_guarantees=[Referer not in ()] + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=25), expr=[l@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0 as k, avg(length(hits.Referer))@1 as l, count(Int64(1))@2 as c, min(hits.Referer)@3 as min(hits.Referer)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: count(Int64(1))@2 > 100000 + │ AggregateExec: mode=FinalPartitioned, gby=[regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0 as regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))], aggr=[avg(length(hits.Referer)), count(Int64(1)), min(hits.Referer)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[regexp_replace(CAST(Referer@0 AS LargeUtf8), ^https?://(?:www\.)?([^/]+)/.*$, \1) as regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))], aggr=[avg(length(hits.Referer)), count(Int64(1)), min(hits.Referer)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: Referer@0 != + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[Referer], file_type=parquet, predicate=Referer@0 != , pruning_predicate=Referer_null_count@2 != row_count@3 AND (Referer_min@0 != OR != Referer_max@1), required_guarantees=[Referer not in ()] + └────────────────────────────────────────────────── "#); Ok(()) } @@ -622,12 +646,12 @@ mod tests { ┌───── DistributedExec ── Tasks: t0:[p0] │ AggregateExec: mode=Final, gby=[], aggr=[sum(hits.ResolutionWidth), sum(hits.ResolutionWidth + Int64(1)), sum(hits.ResolutionWidth + Int64(2)), sum(hits.ResolutionWidth + Int64(3)), sum(hits.ResolutionWidth + Int64(4)), sum(hits.ResolutionWidth + Int64(5)), sum(hits.ResolutionWidth + Int64(6)), sum(hits.ResolutionWidth + Int64(7)), sum(hits.ResolutionWidth + Int64(8)), sum(hits.ResolutionWidth + Int64(9)), sum(hits.ResolutionWidth + Int64(10)), sum(hits.ResolutionWidth + Int64(11)), sum(hits.ResolutionWidth + Int64(12)), sum(hits.ResolutionWidth + Int64(13)), sum(hits.ResolutionWidth + Int64(14)), sum(hits.ResolutionWidth + Int64(15)), sum(hits.ResolutionWidth + Int64(16)), sum(hits.ResolutionWidth + Int64(17)), sum(hits.ResolutionWidth + Int64(18)), sum(hits.ResolutionWidth + Int64(19)), sum(hits.ResolutionWidth + Int64(20)), sum(hits.ResolutionWidth + Int64(21)), sum(hits.ResolutionWidth + Int64(22)), sum(hits.ResolutionWidth + Int64(23)), sum(hits.ResolutionWidth + Int64(24)), sum(hits.ResolutionWidth + Int64(25)), sum(hits.ResolutionWidth + Int64(26)), sum(hits.ResolutionWidth + Int64(27)), sum(hits.ResolutionWidth + Int64(28)), sum(hits.ResolutionWidth + Int64(29)), sum(hits.ResolutionWidth + Int64(30)), sum(hits.ResolutionWidth + Int64(31)), sum(hits.ResolutionWidth + Int64(32)), sum(hits.ResolutionWidth + Int64(33)), sum(hits.ResolutionWidth + Int64(34)), sum(hits.ResolutionWidth + Int64(35)), sum(hits.ResolutionWidth + Int64(36)), sum(hits.ResolutionWidth + Int64(37)), sum(hits.ResolutionWidth + Int64(38)), sum(hits.ResolutionWidth + Int64(39)), sum(hits.ResolutionWidth + Int64(40)), sum(hits.ResolutionWidth + Int64(41)), sum(hits.ResolutionWidth + Int64(42)), sum(hits.ResolutionWidth + Int64(43)), sum(hits.ResolutionWidth + Int64(44)), sum(hits.ResolutionWidth + Int64(45)), sum(hits.ResolutionWidth + Int64(46)), sum(hits.ResolutionWidth + Int64(47)), sum(hits.ResolutionWidth + Int64(48)), sum(hits.ResolutionWidth + Int64(49)), sum(hits.ResolutionWidth + Int64(50)), sum(hits.ResolutionWidth + Int64(51)), sum(hits.ResolutionWidth + Int64(52)), sum(hits.ResolutionWidth + Int64(53)), sum(hits.ResolutionWidth + Int64(54)), sum(hits.ResolutionWidth + Int64(55)), sum(hits.ResolutionWidth + Int64(56)), sum(hits.ResolutionWidth + Int64(57)), sum(hits.ResolutionWidth + Int64(58)), sum(hits.ResolutionWidth + Int64(59)), sum(hits.ResolutionWidth + Int64(60)), sum(hits.ResolutionWidth + Int64(61)), sum(hits.ResolutionWidth + Int64(62)), sum(hits.ResolutionWidth + Int64(63)), sum(hits.ResolutionWidth + Int64(64)), sum(hits.ResolutionWidth + Int64(65)), sum(hits.ResolutionWidth + Int64(66)), sum(hits.ResolutionWidth + Int64(67)), sum(hits.ResolutionWidth + Int64(68)), sum(hits.ResolutionWidth + Int64(69)), sum(hits.ResolutionWidth + Int64(70)), sum(hits.ResolutionWidth + Int64(71)), sum(hits.ResolutionWidth + Int64(72)), sum(hits.ResolutionWidth + Int64(73)), sum(hits.ResolutionWidth + Int64(74)), sum(hits.ResolutionWidth + Int64(75)), sum(hits.ResolutionWidth + Int64(76)), sum(hits.ResolutionWidth + Int64(77)), sum(hits.ResolutionWidth + Int64(78)), sum(hits.ResolutionWidth + Int64(79)), sum(hits.ResolutionWidth + Int64(80)), sum(hits.ResolutionWidth + Int64(81)), sum(hits.ResolutionWidth + Int64(82)), sum(hits.ResolutionWidth + Int64(83)), sum(hits.ResolutionWidth + Int64(84)), sum(hits.ResolutionWidth + Int64(85)), sum(hits.ResolutionWidth + Int64(86)), sum(hits.ResolutionWidth + Int64(87)), sum(hits.ResolutionWidth + Int64(88)), sum(hits.ResolutionWidth + Int64(89))] │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[sum(hits.ResolutionWidth), sum(hits.ResolutionWidth + Int64(1)), sum(hits.ResolutionWidth + Int64(2)), sum(hits.ResolutionWidth + Int64(3)), sum(hits.ResolutionWidth + Int64(4)), sum(hits.ResolutionWidth + Int64(5)), sum(hits.ResolutionWidth + Int64(6)), sum(hits.ResolutionWidth + Int64(7)), sum(hits.ResolutionWidth + Int64(8)), sum(hits.ResolutionWidth + Int64(9)), sum(hits.ResolutionWidth + Int64(10)), sum(hits.ResolutionWidth + Int64(11)), sum(hits.ResolutionWidth + Int64(12)), sum(hits.ResolutionWidth + Int64(13)), sum(hits.ResolutionWidth + Int64(14)), sum(hits.ResolutionWidth + Int64(15)), sum(hits.ResolutionWidth + Int64(16)), sum(hits.ResolutionWidth + Int64(17)), sum(hits.ResolutionWidth + Int64(18)), sum(hits.ResolutionWidth + Int64(19)), sum(hits.ResolutionWidth + Int64(20)), sum(hits.ResolutionWidth + Int64(21)), sum(hits.ResolutionWidth + Int64(22)), sum(hits.ResolutionWidth + Int64(23)), sum(hits.ResolutionWidth + Int64(24)), sum(hits.ResolutionWidth + Int64(25)), sum(hits.ResolutionWidth + Int64(26)), sum(hits.ResolutionWidth + Int64(27)), sum(hits.ResolutionWidth + Int64(28)), sum(hits.ResolutionWidth + Int64(29)), sum(hits.ResolutionWidth + Int64(30)), sum(hits.ResolutionWidth + Int64(31)), sum(hits.ResolutionWidth + Int64(32)), sum(hits.ResolutionWidth + Int64(33)), sum(hits.ResolutionWidth + Int64(34)), sum(hits.ResolutionWidth + Int64(35)), sum(hits.ResolutionWidth + Int64(36)), sum(hits.ResolutionWidth + Int64(37)), sum(hits.ResolutionWidth + Int64(38)), sum(hits.ResolutionWidth + Int64(39)), sum(hits.ResolutionWidth + Int64(40)), sum(hits.ResolutionWidth + Int64(41)), sum(hits.ResolutionWidth + Int64(42)), sum(hits.ResolutionWidth + Int64(43)), sum(hits.ResolutionWidth + Int64(44)), sum(hits.ResolutionWidth + Int64(45)), sum(hits.ResolutionWidth + Int64(46)), sum(hits.ResolutionWidth + Int64(47)), sum(hits.ResolutionWidth + Int64(48)), sum(hits.ResolutionWidth + Int64(49)), sum(hits.ResolutionWidth + Int64(50)), sum(hits.ResolutionWidth + Int64(51)), sum(hits.ResolutionWidth + Int64(52)), sum(hits.ResolutionWidth + Int64(53)), sum(hits.ResolutionWidth + Int64(54)), sum(hits.ResolutionWidth + Int64(55)), sum(hits.ResolutionWidth + Int64(56)), sum(hits.ResolutionWidth + Int64(57)), sum(hits.ResolutionWidth + Int64(58)), sum(hits.ResolutionWidth + Int64(59)), sum(hits.ResolutionWidth + Int64(60)), sum(hits.ResolutionWidth + Int64(61)), sum(hits.ResolutionWidth + Int64(62)), sum(hits.ResolutionWidth + Int64(63)), sum(hits.ResolutionWidth + Int64(64)), sum(hits.ResolutionWidth + Int64(65)), sum(hits.ResolutionWidth + Int64(66)), sum(hits.ResolutionWidth + Int64(67)), sum(hits.ResolutionWidth + Int64(68)), sum(hits.ResolutionWidth + Int64(69)), sum(hits.ResolutionWidth + Int64(70)), sum(hits.ResolutionWidth + Int64(71)), sum(hits.ResolutionWidth + Int64(72)), sum(hits.ResolutionWidth + Int64(73)), sum(hits.ResolutionWidth + Int64(74)), sum(hits.ResolutionWidth + Int64(75)), sum(hits.ResolutionWidth + Int64(76)), sum(hits.ResolutionWidth + Int64(77)), sum(hits.ResolutionWidth + Int64(78)), sum(hits.ResolutionWidth + Int64(79)), sum(hits.ResolutionWidth + Int64(80)), sum(hits.ResolutionWidth + Int64(81)), sum(hits.ResolutionWidth + Int64(82)), sum(hits.ResolutionWidth + Int64(83)), sum(hits.ResolutionWidth + Int64(84)), sum(hits.ResolutionWidth + Int64(85)), sum(hits.ResolutionWidth + Int64(86)), sum(hits.ResolutionWidth + Int64(87)), sum(hits.ResolutionWidth + Int64(88)), sum(hits.ResolutionWidth + Int64(89))] │ ProjectionExec: expr=[CAST(ResolutionWidth@0 AS Int64) as __common_expr_1] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[ResolutionWidth], file_type=parquet └────────────────────────────────────────────────── "); @@ -640,20 +664,23 @@ mod tests { assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [c@2 DESC], fetch=10 - │ SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[SearchEngineID@0 as SearchEngineID, ClientIP@1 as ClientIP, count(Int64(1))@2 as c, sum(hits.IsRefresh)@3 as sum(hits.IsRefresh), avg(hits.ResolutionWidth)@4 as avg(hits.ResolutionWidth)] - │ AggregateExec: mode=FinalPartitioned, gby=[SearchEngineID@0 as SearchEngineID, ClientIP@1 as ClientIP], aggr=[count(Int64(1)), sum(hits.IsRefresh), avg(hits.ResolutionWidth)] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([SearchEngineID@0, ClientIP@1], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[SearchEngineID@3 as SearchEngineID, ClientIP@0 as ClientIP], aggr=[count(Int64(1)), sum(hits.IsRefresh), avg(hits.ResolutionWidth)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: SearchPhrase@4 != , projection=[ClientIP@0, IsRefresh@1, ResolutionWidth@2, SearchEngineID@3] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] - │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[ClientIP, IsRefresh, ResolutionWidth, SearchEngineID, SearchPhrase], file_type=parquet, predicate=SearchPhrase@4 != , pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[SearchEngineID@0 as SearchEngineID, ClientIP@1 as ClientIP, count(Int64(1))@2 as c, sum(hits.IsRefresh)@3 as sum(hits.IsRefresh), avg(hits.ResolutionWidth)@4 as avg(hits.ResolutionWidth)] + │ AggregateExec: mode=FinalPartitioned, gby=[SearchEngineID@0 as SearchEngineID, ClientIP@1 as ClientIP], aggr=[count(Int64(1)), sum(hits.IsRefresh), avg(hits.ResolutionWidth)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([SearchEngineID@0, ClientIP@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[SearchEngineID@3 as SearchEngineID, ClientIP@0 as ClientIP], aggr=[count(Int64(1)), sum(hits.IsRefresh), avg(hits.ResolutionWidth)] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: SearchPhrase@4 != , projection=[ClientIP@0, IsRefresh@1, ResolutionWidth@2, SearchEngineID@3] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[ClientIP, IsRefresh, ResolutionWidth, SearchEngineID, SearchPhrase], file_type=parquet, predicate=SearchPhrase@4 != , pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + └────────────────────────────────────────────────── "); Ok(()) } @@ -686,13 +713,13 @@ mod tests { │ SortExec: TopK(fetch=10), expr=[c@1 DESC], preserve_partitioning=[true] │ ProjectionExec: expr=[URL@0 as URL, count(Int64(1))@1 as c] │ AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([URL@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([URL@0], 6), input_partitions=2 │ AggregateExec: mode=Partial, gby=[URL@0 as URL], aggr=[count(Int64(1))] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[URL], file_type=parquet └────────────────────────────────────────────────── "); @@ -711,13 +738,13 @@ mod tests { │ SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true] │ ProjectionExec: expr=[1 as Int64(1), URL@0 as URL, count(Int64(1))@1 as c] │ AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([URL@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([URL@0], 6), input_partitions=2 │ AggregateExec: mode=Partial, gby=[URL@0 as URL], aggr=[count(Int64(1))] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[URL], file_type=parquet └────────────────────────────────────────────────── "); @@ -736,14 +763,14 @@ mod tests { │ SortExec: TopK(fetch=10), expr=[c@4 DESC], preserve_partitioning=[true] │ ProjectionExec: expr=[ClientIP@0 as ClientIP, hits.ClientIP - Int64(1)@1 as hits.ClientIP - Int64(1), hits.ClientIP - Int64(2)@2 as hits.ClientIP - Int64(2), hits.ClientIP - Int64(3)@3 as hits.ClientIP - Int64(3), count(Int64(1))@4 as c] │ AggregateExec: mode=FinalPartitioned, gby=[ClientIP@0 as ClientIP, hits.ClientIP - Int64(1)@1 as hits.ClientIP - Int64(1), hits.ClientIP - Int64(2)@2 as hits.ClientIP - Int64(2), hits.ClientIP - Int64(3)@3 as hits.ClientIP - Int64(3)], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ClientIP@0, hits.ClientIP - Int64(1)@1, hits.ClientIP - Int64(2)@2, hits.ClientIP - Int64(3)@3], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([ClientIP@0, hits.ClientIP - Int64(1)@1, hits.ClientIP - Int64(2)@2, hits.ClientIP - Int64(3)@3], 6), input_partitions=2 │ AggregateExec: mode=Partial, gby=[ClientIP@1 as ClientIP, __common_expr_1@0 - 1 as hits.ClientIP - Int64(1), __common_expr_1@0 - 2 as hits.ClientIP - Int64(2), __common_expr_1@0 - 3 as hits.ClientIP - Int64(3)], aggr=[count(Int64(1))] │ ProjectionExec: expr=[CAST(ClientIP@0 AS Int64) as __common_expr_1, ClientIP@0 as ClientIP] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[ClientIP], file_type=parquet └────────────────────────────────────────────────── "); @@ -756,20 +783,23 @@ mod tests { assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [pageviews@1 DESC], fetch=10 - │ SortExec: TopK(fetch=10), expr=[pageviews@1 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[URL@0 as URL, count(Int64(1))@1 as pageviews] - │ AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([URL@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[URL@0 as URL], aggr=[count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND DontCountHits@4 = 0 AND IsRefresh@3 = 0 AND URL@2 != , projection=[URL@2] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] - │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventDate, CounterID, URL, IsRefresh, DontCountHits], file_type=parquet, predicate=CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND DontCountHits@4 = 0 AND IsRefresh@3 = 0 AND URL@2 != , pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND DontCountHits_null_count@6 != row_count@3 AND DontCountHits_min@4 <= 0 AND 0 <= DontCountHits_max@5 AND IsRefresh_null_count@9 != row_count@3 AND IsRefresh_min@7 <= 0 AND 0 <= IsRefresh_max@8 AND URL_null_count@12 != row_count@3 AND (URL_min@10 != OR != URL_max@11), required_guarantees=[CounterID in (62), DontCountHits in (0), IsRefresh in (0), URL not in ()] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[pageviews@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[URL@0 as URL, count(Int64(1))@1 as pageviews] + │ AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([URL@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND DontCountHits@4 = 0 AND IsRefresh@3 = 0 AND URL@2 != , projection=[URL@2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventDate, CounterID, URL, IsRefresh, DontCountHits], file_type=parquet, predicate=CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND DontCountHits@4 = 0 AND IsRefresh@3 = 0 AND URL@2 != , pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND DontCountHits_null_count@6 != row_count@3 AND DontCountHits_min@4 <= 0 AND 0 <= DontCountHits_max@5 AND IsRefresh_null_count@9 != row_count@3 AND IsRefresh_min@7 <= 0 AND 0 <= IsRefresh_max@8 AND URL_null_count@12 != row_count@3 AND (URL_min@10 != OR != URL_max@11), required_guarantees=[CounterID in (62), DontCountHits in (0), IsRefresh in (0), URL not in ()] + └────────────────────────────────────────────────── "); Ok(()) } @@ -780,20 +810,23 @@ mod tests { assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [pageviews@1 DESC], fetch=10 - │ SortExec: TopK(fetch=10), expr=[pageviews@1 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[Title@0 as Title, count(Int64(1))@1 as pageviews] - │ AggregateExec: mode=FinalPartitioned, gby=[Title@0 as Title], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([Title@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[Title@0 as Title], aggr=[count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: CounterID@2 = 62 AND CAST(EventDate@1 AS Utf8) >= 2013-07-01 AND CAST(EventDate@1 AS Utf8) <= 2013-07-31 AND DontCountHits@4 = 0 AND IsRefresh@3 = 0 AND Title@0 != , projection=[Title@0] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] - │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[Title, EventDate, CounterID, IsRefresh, DontCountHits], file_type=parquet, predicate=CounterID@2 = 62 AND CAST(EventDate@1 AS Utf8) >= 2013-07-01 AND CAST(EventDate@1 AS Utf8) <= 2013-07-31 AND DontCountHits@4 = 0 AND IsRefresh@3 = 0 AND Title@0 != , pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND DontCountHits_null_count@6 != row_count@3 AND DontCountHits_min@4 <= 0 AND 0 <= DontCountHits_max@5 AND IsRefresh_null_count@9 != row_count@3 AND IsRefresh_min@7 <= 0 AND 0 <= IsRefresh_max@8 AND Title_null_count@12 != row_count@3 AND (Title_min@10 != OR != Title_max@11), required_guarantees=[CounterID in (62), DontCountHits in (0), IsRefresh in (0), Title not in ()] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[pageviews@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[Title@0 as Title, count(Int64(1))@1 as pageviews] + │ AggregateExec: mode=FinalPartitioned, gby=[Title@0 as Title], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([Title@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[Title@0 as Title], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CounterID@2 = 62 AND CAST(EventDate@1 AS Utf8) >= 2013-07-01 AND CAST(EventDate@1 AS Utf8) <= 2013-07-31 AND DontCountHits@4 = 0 AND IsRefresh@3 = 0 AND Title@0 != , projection=[Title@0] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[Title, EventDate, CounterID, IsRefresh, DontCountHits], file_type=parquet, predicate=CounterID@2 = 62 AND CAST(EventDate@1 AS Utf8) >= 2013-07-01 AND CAST(EventDate@1 AS Utf8) <= 2013-07-31 AND DontCountHits@4 = 0 AND IsRefresh@3 = 0 AND Title@0 != , pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND DontCountHits_null_count@6 != row_count@3 AND DontCountHits_min@4 <= 0 AND 0 <= DontCountHits_max@5 AND IsRefresh_null_count@9 != row_count@3 AND IsRefresh_min@7 <= 0 AND 0 <= IsRefresh_max@8 AND Title_null_count@12 != row_count@3 AND (Title_min@10 != OR != Title_max@11), required_guarantees=[CounterID in (62), DontCountHits in (0), IsRefresh in (0), Title not in ()] + └────────────────────────────────────────────────── "); Ok(()) } @@ -805,20 +838,23 @@ mod tests { ┌───── DistributedExec ── Tasks: t0:[p0] │ GlobalLimitExec: skip=1000, fetch=10 │ SortPreservingMergeExec: [pageviews@1 DESC], fetch=1010 - │ SortExec: TopK(fetch=1010), expr=[pageviews@1 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[URL@0 as URL, count(Int64(1))@1 as pageviews] - │ AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([URL@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[URL@0 as URL], aggr=[count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@3 = 0 AND IsLink@4 != 0 AND IsDownload@5 = 0, projection=[URL@2] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] - │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventDate, CounterID, URL, IsRefresh, IsLink, IsDownload], file_type=parquet, predicate=CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@3 = 0 AND IsLink@4 != 0 AND IsDownload@5 = 0, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5 AND IsLink_null_count@9 != row_count@3 AND (IsLink_min@7 != 0 OR 0 != IsLink_max@8) AND IsDownload_null_count@12 != row_count@3 AND IsDownload_min@10 <= 0 AND 0 <= IsDownload_max@11, required_guarantees=[CounterID in (62), IsDownload in (0), IsLink not in (0), IsRefresh in (0)] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=1010), expr=[pageviews@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[URL@0 as URL, count(Int64(1))@1 as pageviews] + │ AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([URL@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@3 = 0 AND IsLink@4 != 0 AND IsDownload@5 = 0, projection=[URL@2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventDate, CounterID, URL, IsRefresh, IsLink, IsDownload], file_type=parquet, predicate=CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@3 = 0 AND IsLink@4 != 0 AND IsDownload@5 = 0, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5 AND IsLink_null_count@9 != row_count@3 AND (IsLink_min@7 != 0 OR 0 != IsLink_max@8) AND IsDownload_null_count@12 != row_count@3 AND IsDownload_min@10 <= 0 AND 0 <= IsDownload_max@11, required_guarantees=[CounterID in (62), IsDownload in (0), IsLink not in (0), IsRefresh in (0)] + └────────────────────────────────────────────────── "); Ok(()) } @@ -830,20 +866,23 @@ mod tests { ┌───── DistributedExec ── Tasks: t0:[p0] │ GlobalLimitExec: skip=1000, fetch=10 │ SortPreservingMergeExec: [pageviews@5 DESC], fetch=1010 - │ SortExec: TopK(fetch=1010), expr=[pageviews@5 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[TraficSourceID@0 as TraficSourceID, SearchEngineID@1 as SearchEngineID, AdvEngineID@2 as AdvEngineID, CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END@3 as src, URL@4 as dst, count(Int64(1))@5 as pageviews] - │ AggregateExec: mode=FinalPartitioned, gby=[TraficSourceID@0 as TraficSourceID, SearchEngineID@1 as SearchEngineID, AdvEngineID@2 as AdvEngineID, CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END@3 as CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END, URL@4 as URL], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([TraficSourceID@0, SearchEngineID@1, AdvEngineID@2, CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END@3, URL@4], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[TraficSourceID@2 as TraficSourceID, SearchEngineID@3 as SearchEngineID, AdvEngineID@4 as AdvEngineID, CASE WHEN SearchEngineID@3 = 0 AND AdvEngineID@4 = 0 THEN Referer@1 ELSE END as CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END, URL@0 as URL], aggr=[count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@4 = 0, projection=[URL@2, Referer@3, TraficSourceID@5, SearchEngineID@6, AdvEngineID@7] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] - │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventDate, CounterID, URL, Referer, IsRefresh, TraficSourceID, SearchEngineID, AdvEngineID], file_type=parquet, predicate=CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@4 = 0, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5, required_guarantees=[CounterID in (62), IsRefresh in (0)] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=1010), expr=[pageviews@5 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[TraficSourceID@0 as TraficSourceID, SearchEngineID@1 as SearchEngineID, AdvEngineID@2 as AdvEngineID, CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END@3 as src, URL@4 as dst, count(Int64(1))@5 as pageviews] + │ AggregateExec: mode=FinalPartitioned, gby=[TraficSourceID@0 as TraficSourceID, SearchEngineID@1 as SearchEngineID, AdvEngineID@2 as AdvEngineID, CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END@3 as CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END, URL@4 as URL], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([TraficSourceID@0, SearchEngineID@1, AdvEngineID@2, CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END@3, URL@4], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[TraficSourceID@2 as TraficSourceID, SearchEngineID@3 as SearchEngineID, AdvEngineID@4 as AdvEngineID, CASE WHEN SearchEngineID@3 = 0 AND AdvEngineID@4 = 0 THEN Referer@1 ELSE END as CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END, URL@0 as URL], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@4 = 0, projection=[URL@2, Referer@3, TraficSourceID@5, SearchEngineID@6, AdvEngineID@7] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventDate, CounterID, URL, Referer, IsRefresh, TraficSourceID, SearchEngineID, AdvEngineID], file_type=parquet, predicate=CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@4 = 0, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5, required_guarantees=[CounterID in (62), IsRefresh in (0)] + └────────────────────────────────────────────────── "#); Ok(()) } @@ -855,20 +894,23 @@ mod tests { ┌───── DistributedExec ── Tasks: t0:[p0] │ GlobalLimitExec: skip=100, fetch=10 │ SortPreservingMergeExec: [pageviews@2 DESC], fetch=110 - │ SortExec: TopK(fetch=110), expr=[pageviews@2 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[URLHash@0 as URLHash, EventDate@1 as EventDate, count(Int64(1))@2 as pageviews] - │ AggregateExec: mode=FinalPartitioned, gby=[URLHash@0 as URLHash, EventDate@1 as EventDate], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([URLHash@0, EventDate@1], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[URLHash@1 as URLHash, EventDate@0 as EventDate], aggr=[count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@2 = 0 AND (TraficSourceID@3 = -1 OR TraficSourceID@3 = 6) AND RefererHash@4 = 3594120000172545465, projection=[EventDate@0, URLHash@5] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] - │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventDate, CounterID, IsRefresh, TraficSourceID, RefererHash, URLHash], file_type=parquet, predicate=CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@2 = 0 AND (TraficSourceID@3 = -1 OR TraficSourceID@3 = 6) AND RefererHash@4 = 3594120000172545465, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5 AND (TraficSourceID_null_count@9 != row_count@3 AND TraficSourceID_min@7 <= -1 AND -1 <= TraficSourceID_max@8 OR TraficSourceID_null_count@9 != row_count@3 AND TraficSourceID_min@7 <= 6 AND 6 <= TraficSourceID_max@8) AND RefererHash_null_count@12 != row_count@3 AND RefererHash_min@10 <= 3594120000172545465 AND 3594120000172545465 <= RefererHash_max@11, required_guarantees=[CounterID in (62), IsRefresh in (0), RefererHash in (3594120000172545465), TraficSourceID in (-1, 6)] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=110), expr=[pageviews@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[URLHash@0 as URLHash, EventDate@1 as EventDate, count(Int64(1))@2 as pageviews] + │ AggregateExec: mode=FinalPartitioned, gby=[URLHash@0 as URLHash, EventDate@1 as EventDate], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([URLHash@0, EventDate@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[URLHash@1 as URLHash, EventDate@0 as EventDate], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@2 = 0 AND (TraficSourceID@3 = -1 OR TraficSourceID@3 = 6) AND RefererHash@4 = 3594120000172545465, projection=[EventDate@0, URLHash@5] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventDate, CounterID, IsRefresh, TraficSourceID, RefererHash, URLHash], file_type=parquet, predicate=CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@2 = 0 AND (TraficSourceID@3 = -1 OR TraficSourceID@3 = 6) AND RefererHash@4 = 3594120000172545465, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5 AND (TraficSourceID_null_count@9 != row_count@3 AND TraficSourceID_min@7 <= -1 AND -1 <= TraficSourceID_max@8 OR TraficSourceID_null_count@9 != row_count@3 AND TraficSourceID_min@7 <= 6 AND 6 <= TraficSourceID_max@8) AND RefererHash_null_count@12 != row_count@3 AND RefererHash_min@10 <= 3594120000172545465 AND 3594120000172545465 <= RefererHash_max@11, required_guarantees=[CounterID in (62), IsRefresh in (0), RefererHash in (3594120000172545465), TraficSourceID in (-1, 6)] + └────────────────────────────────────────────────── "); Ok(()) } @@ -880,20 +922,23 @@ mod tests { ┌───── DistributedExec ── Tasks: t0:[p0] │ GlobalLimitExec: skip=10000, fetch=10 │ SortPreservingMergeExec: [pageviews@2 DESC], fetch=10010 - │ SortExec: TopK(fetch=10010), expr=[pageviews@2 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[WindowClientWidth@0 as WindowClientWidth, WindowClientHeight@1 as WindowClientHeight, count(Int64(1))@2 as pageviews] - │ AggregateExec: mode=FinalPartitioned, gby=[WindowClientWidth@0 as WindowClientWidth, WindowClientHeight@1 as WindowClientHeight], aggr=[count(Int64(1))] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([WindowClientWidth@0, WindowClientHeight@1], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[WindowClientWidth@0 as WindowClientWidth, WindowClientHeight@1 as WindowClientHeight], aggr=[count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@2 = 0 AND DontCountHits@5 = 0 AND URLHash@6 = 2868770270353813622, projection=[WindowClientWidth@3, WindowClientHeight@4] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] - │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventDate, CounterID, IsRefresh, WindowClientWidth, WindowClientHeight, DontCountHits, URLHash], file_type=parquet, predicate=CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@2 = 0 AND DontCountHits@5 = 0 AND URLHash@6 = 2868770270353813622, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5 AND DontCountHits_null_count@9 != row_count@3 AND DontCountHits_min@7 <= 0 AND 0 <= DontCountHits_max@8 AND URLHash_null_count@12 != row_count@3 AND URLHash_min@10 <= 2868770270353813622 AND 2868770270353813622 <= URLHash_max@11, required_guarantees=[CounterID in (62), DontCountHits in (0), IsRefresh in (0), URLHash in (2868770270353813622)] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10010), expr=[pageviews@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[WindowClientWidth@0 as WindowClientWidth, WindowClientHeight@1 as WindowClientHeight, count(Int64(1))@2 as pageviews] + │ AggregateExec: mode=FinalPartitioned, gby=[WindowClientWidth@0 as WindowClientWidth, WindowClientHeight@1 as WindowClientHeight], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([WindowClientWidth@0, WindowClientHeight@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[WindowClientWidth@0 as WindowClientWidth, WindowClientHeight@1 as WindowClientHeight], aggr=[count(Int64(1))] + │ CoalesceBatchesExec: target_batch_size=8192 + │ FilterExec: CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@2 = 0 AND DontCountHits@5 = 0 AND URLHash@6 = 2868770270353813622, projection=[WindowClientWidth@3, WindowClientHeight@4] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventDate, CounterID, IsRefresh, WindowClientWidth, WindowClientHeight, DontCountHits, URLHash], file_type=parquet, predicate=CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@2 = 0 AND DontCountHits@5 = 0 AND URLHash@6 = 2868770270353813622, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5 AND DontCountHits_null_count@9 != row_count@3 AND DontCountHits_min@7 <= 0 AND 0 <= DontCountHits_max@8 AND URLHash_null_count@12 != row_count@3 AND URLHash_min@10 <= 2868770270353813622 AND 2868770270353813622 <= URLHash_max@11, required_guarantees=[CounterID in (62), DontCountHits in (0), IsRefresh in (0), URLHash in (2868770270353813622)] + └────────────────────────────────────────────────── "); Ok(()) } diff --git a/tests/tpcds_plans_test.rs b/tests/tpcds_plans_test.rs index d117bda7..5c80adac 100644 --- a/tests/tpcds_plans_test.rs +++ b/tests/tpcds_plans_test.rs @@ -260,7 +260,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ws_sold_date_sk@9, ws_ext_discount_amt@11, ws_ext_sales_price@12, ws_ext_wholesale_cost@13, ws_ext_list_price@14] │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], filter=CASE WHEN year_total@2 > Some(0),24,6 THEN year_total@3 / year_total@2 END > CASE WHEN year_total@0 > Some(0),24,6 THEN year_total@1 / year_total@0 END, projection=[customer_id@0, customer_id@2, customer_first_name@3, customer_last_name@4, customer_preferred_cust_flag@5, year_total@7, year_total@9] │ CoalescePartitionsExec @@ -284,7 +284,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, cs_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, cs_sold_date_sk@9, cs_ext_discount_amt@11, cs_ext_sales_price@12, cs_ext_wholesale_cost@13, cs_ext_list_price@14] │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)] │ CoalescePartitionsExec @@ -304,7 +304,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ss_sold_date_sk@9, ss_ext_discount_amt@11, ss_ext_sales_price@12, ss_ext_wholesale_cost@13, ss_ext_list_price@14] │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[c_customer_id@0 as customer_id, c_first_name@1 as customer_first_name, c_last_name@2 as customer_last_name, c_preferred_cust_flag@3 as customer_preferred_cust_flag, sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))@8 as year_total] │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) │ CoalesceBatchesExec: target_batch_size=8192 @@ -318,7 +318,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ss_sold_date_sk@9, ss_ext_discount_amt@11, ss_ext_sales_price@12, ss_ext_wholesale_cost@13, ss_ext_list_price@14] │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))@8 as year_total] │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) │ CoalesceBatchesExec: target_batch_size=8192 @@ -332,7 +332,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, cs_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, cs_sold_date_sk@9, cs_ext_discount_amt@11, cs_ext_sales_price@12, cs_ext_wholesale_cost@13, cs_ext_list_price@14] │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))@8 as year_total] │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) │ CoalesceBatchesExec: target_batch_size=8192 @@ -346,7 +346,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ws_sold_date_sk@9, ws_ext_discount_amt@11, ws_ext_sales_price@12, ws_ext_wholesale_cost@13, ws_ext_list_price@14] │ [Stage 17] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 18] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 18] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] @@ -364,10 +364,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -386,10 +386,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -408,10 +408,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -430,10 +430,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 13 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -452,10 +452,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 15 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 15 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -474,10 +474,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 18 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 18 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); @@ -545,7 +545,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(wr_item_sk@1, ws_item_sk@0), (wr_order_number@2, ws_order_number@2)] │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_id@1 as s_store_id, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] @@ -618,10 +618,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_item_sk@0, ws_order_number@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ws_item_sk@0, ws_order_number@2], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_item_sk, ws_web_site_sk, ws_order_number], file_type=parquet └────────────────────────────────────────────────── "); @@ -739,8 +739,8 @@ mod tests { │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, ss_cdemo_sk@2)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_promo_sk@5, ss_quantity@6, ss_list_price@7, ss_sales_price@8, ss_coupon_amt@9] - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── @@ -760,19 +760,19 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=3 + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=2 │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_promo_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); @@ -881,53 +881,53 @@ mod tests { │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] │ CoalescePartitionsExec - │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_net_paid)] │ CoalescePartitionsExec - │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] │ CoalescePartitionsExec - │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] │ CoalescePartitionsExec - │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_net_paid)] │ CoalescePartitionsExec - │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] │ CoalescePartitionsExec - │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] │ CoalescePartitionsExec - │ [Stage 9] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 9] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_net_paid)] │ CoalescePartitionsExec - │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] │ CoalescePartitionsExec - │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] │ CoalescePartitionsExec - │ [Stage 12] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 12] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_net_paid)] │ CoalescePartitionsExec - │ [Stage 13] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 13] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] │ CoalescePartitionsExec - │ [Stage 14] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 14] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] │ CoalescePartitionsExec - │ [Stage 15] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 15] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_net_paid)] │ CoalescePartitionsExec - │ [Stage 16] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 16] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[] @@ -937,114 +937,114 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/reason/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-3.parquet]]}, projection=[r_reason_sk], file_type=parquet, predicate=r_reason_sk@0 = 1, pruning_predicate=r_reason_sk_null_count@2 != row_count@3 AND r_reason_sk_min@0 <= 1 AND 1 <= r_reason_sk_max@1, required_guarantees=[r_reason_sk in (1)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] │ ProjectionExec: expr=[] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 1 AND ss_quantity@0 <= 20 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity], file_type=parquet, predicate=ss_quantity@0 >= 1 AND ss_quantity@0 <= 20, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 1 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 20, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 3 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 1 AND ss_quantity@0 <= 20, projection=[ss_ext_discount_amt@1] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_ext_discount_amt], file_type=parquet, predicate=ss_quantity@0 >= 1 AND ss_quantity@0 <= 20, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 1 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 20, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 4 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_net_paid)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 1 AND ss_quantity@0 <= 20, projection=[ss_net_paid@1] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_net_paid], file_type=parquet, predicate=ss_quantity@0 >= 1 AND ss_quantity@0 <= 20, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 1 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 20, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 5 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] │ ProjectionExec: expr=[] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 21 AND ss_quantity@0 <= 40 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity], file_type=parquet, predicate=ss_quantity@0 >= 21 AND ss_quantity@0 <= 40, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 21 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 40, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 6 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 21 AND ss_quantity@0 <= 40, projection=[ss_ext_discount_amt@1] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_ext_discount_amt], file_type=parquet, predicate=ss_quantity@0 >= 21 AND ss_quantity@0 <= 40, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 21 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 40, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 7 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_net_paid)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 21 AND ss_quantity@0 <= 40, projection=[ss_net_paid@1] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_net_paid], file_type=parquet, predicate=ss_quantity@0 >= 21 AND ss_quantity@0 <= 40, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 21 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 40, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 8 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] │ ProjectionExec: expr=[] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 41 AND ss_quantity@0 <= 60 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity], file_type=parquet, predicate=ss_quantity@0 >= 41 AND ss_quantity@0 <= 60, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 41 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 60, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 9 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 41 AND ss_quantity@0 <= 60, projection=[ss_ext_discount_amt@1] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_ext_discount_amt], file_type=parquet, predicate=ss_quantity@0 >= 41 AND ss_quantity@0 <= 60, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 41 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 60, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 10 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_net_paid)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 41 AND ss_quantity@0 <= 60, projection=[ss_net_paid@1] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_net_paid], file_type=parquet, predicate=ss_quantity@0 >= 41 AND ss_quantity@0 <= 60, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 41 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 60, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 11 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] │ ProjectionExec: expr=[] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 61 AND ss_quantity@0 <= 80 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity], file_type=parquet, predicate=ss_quantity@0 >= 61 AND ss_quantity@0 <= 80, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 61 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 80, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 12 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 61 AND ss_quantity@0 <= 80, projection=[ss_ext_discount_amt@1] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_ext_discount_amt], file_type=parquet, predicate=ss_quantity@0 >= 61 AND ss_quantity@0 <= 80, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 61 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 80, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 13 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 13 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_net_paid)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 61 AND ss_quantity@0 <= 80, projection=[ss_net_paid@1] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_net_paid], file_type=parquet, predicate=ss_quantity@0 >= 61 AND ss_quantity@0 <= 80, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 61 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 80, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 14 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] │ ProjectionExec: expr=[] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 81 AND ss_quantity@0 <= 100 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity], file_type=parquet, predicate=ss_quantity@0 >= 81 AND ss_quantity@0 <= 100, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 81 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 100, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 15 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 15 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 81 AND ss_quantity@0 <= 100, projection=[ss_ext_discount_amt@1] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_ext_discount_amt], file_type=parquet, predicate=ss_quantity@0 >= 81 AND ss_quantity@0 <= 100, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 81 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 100, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 16 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_net_paid)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 81 AND ss_quantity@0 <= 100, projection=[ss_net_paid@1] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_net_paid], file_type=parquet, predicate=ss_quantity@0 >= 81 AND ss_quantity@0 <= 100, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 81 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 100, required_guarantees=[] └────────────────────────────────────────────────── "); @@ -1167,7 +1167,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ws_sold_date_sk@9, ws_ext_discount_amt@11, ws_ext_list_price@12] │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)] │ CoalescePartitionsExec @@ -1187,7 +1187,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ss_sold_date_sk@9, ss_ext_discount_amt@11, ss_ext_list_price@12] │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[c_customer_id@0 as customer_id, c_first_name@1 as customer_first_name, c_last_name@2 as customer_last_name, c_preferred_cust_flag@3 as customer_preferred_cust_flag, sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)@8 as year_total] │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)], ordering_mode=PartiallySorted([7]) │ CoalesceBatchesExec: target_batch_size=8192 @@ -1201,7 +1201,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ss_sold_date_sk@9, ss_ext_discount_amt@11, ss_ext_list_price@12] │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)@8 as year_total] │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)], ordering_mode=PartiallySorted([7]) │ CoalesceBatchesExec: target_batch_size=8192 @@ -1215,7 +1215,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ws_sold_date_sk@9, ws_ext_discount_amt@11, ws_ext_list_price@12] │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] @@ -1233,10 +1233,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -1255,10 +1255,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -1277,10 +1277,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -1299,10 +1299,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); @@ -1447,7 +1447,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ss_sold_date_sk@4, ss_item_sk@5, ss_quantity@6, ss_list_price@7] │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[i_item_sk@0 as ss_item_sk] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(brand_id@0, i_brand_id@1), (class_id@1, i_class_id@2), (category_id@2, i_category_id@3)], projection=[i_item_sk@3] @@ -1463,7 +1463,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ws_sold_date_sk@4] │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ AggregateExec: mode=SinglePartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true @@ -1476,7 +1476,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, cs_sold_date_sk@4] │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ AggregateExec: mode=FinalPartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 3), input_partitions=3 @@ -1490,7 +1490,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ss_sold_date_sk@4] │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] │ ProjectionExec: expr=[catalog as channel, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)@3 as sales, count(Int64(1))@4 as number_sales] @@ -1515,7 +1515,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, cs_sold_date_sk@4, cs_item_sk@5, cs_quantity@6, cs_list_price@7] │ [Stage 22] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 23] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 23] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[i_item_sk@0 as ss_item_sk] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(brand_id@0, i_brand_id@1), (class_id@1, i_class_id@2), (category_id@2, i_category_id@3)], projection=[i_item_sk@3] @@ -1531,7 +1531,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ws_sold_date_sk@4] │ [Stage 25] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 26] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 26] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ AggregateExec: mode=SinglePartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true @@ -1544,7 +1544,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, cs_sold_date_sk@4] │ [Stage 28] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 29] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 29] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ AggregateExec: mode=FinalPartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 3), input_partitions=3 @@ -1558,7 +1558,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ss_sold_date_sk@4] │ [Stage 31] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 32] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 32] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] │ ProjectionExec: expr=[web as channel, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, sum(web_sales.ws_quantity * web_sales.ws_list_price)@3 as sales, count(Int64(1))@4 as number_sales] @@ -1583,7 +1583,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ws_sold_date_sk@4, ws_item_sk@5, ws_quantity@6, ws_list_price@7] │ [Stage 38] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 39] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 39] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[i_item_sk@0 as ss_item_sk] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(brand_id@0, i_brand_id@1), (class_id@1, i_class_id@2), (category_id@2, i_category_id@3)], projection=[i_item_sk@3] @@ -1599,7 +1599,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ws_sold_date_sk@4] │ [Stage 41] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 42] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 42] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ AggregateExec: mode=SinglePartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true @@ -1612,7 +1612,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, cs_sold_date_sk@4] │ [Stage 44] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 45] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 45] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ AggregateExec: mode=FinalPartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 3), input_partitions=3 @@ -1626,7 +1626,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ss_sold_date_sk@4] │ [Stage 47] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 48] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 48] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── @@ -1691,10 +1691,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -1712,10 +1712,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -1733,10 +1733,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 13 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 13 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -1754,10 +1754,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 20 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] @@ -1821,10 +1821,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 23 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 23 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 24 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -1842,10 +1842,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 26 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 26 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 27 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -1863,10 +1863,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 29 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 29 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 30 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -1884,10 +1884,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 32 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 32 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 36 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] @@ -1951,10 +1951,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 39 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 39 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 40 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -1972,10 +1972,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 42 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 42 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 43 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -1993,10 +1993,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 45 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 45 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 46 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -2014,10 +2014,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 48 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 48 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); @@ -2042,7 +2042,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@2)], filter=substr(ca_zip@2, 1, 5) IN ([85669, 86197, 88274, 83405, 86475, 85392, 85460, 80348, 81792]) OR ca_state@1 = CA OR ca_state@1 = WA OR ca_state@1 = GA OR cs_sales_price@0 > Some(50000),5,2, projection=[ca_zip@2, cs_sold_date_sk@3, cs_sales_price@4] │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] @@ -2059,27 +2059,27 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state, ca_zip], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([c_current_addr_sk@2], 3), input_partitions=3 │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_sales_price@2 as cs_sales_price, c_current_addr_sk@0 as c_current_addr_sk] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@2, cs_bill_customer_sk@1)], projection=[c_current_addr_sk@1, cs_sold_date_sk@3, cs_sales_price@5] │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@2], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@2], 9), input_partitions=3 │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); @@ -2183,8 +2183,8 @@ mod tests { │ ProjectionExec: expr=[ss_sold_date_sk@2 as ss_sold_date_sk, ss_item_sk@3 as ss_item_sk, ss_store_sk@4 as ss_store_sk, ss_quantity@5 as ss_quantity, sr_returned_date_sk@6 as sr_returned_date_sk, sr_return_quantity@7 as sr_return_quantity, cs_sold_date_sk@0 as cs_sold_date_sk, cs_quantity@1 as cs_quantity] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cs_bill_customer_sk@1, sr_customer_sk@6), (cs_item_sk@2, sr_item_sk@5)], projection=[cs_sold_date_sk@0, cs_quantity@3, ss_sold_date_sk@4, ss_item_sk@5, ss_store_sk@6, ss_quantity@7, sr_returned_date_sk@8, sr_return_quantity@11] - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── @@ -2217,31 +2217,31 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@1 = 2001Q1, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1)] └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1, cs_item_sk@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1, cs_item_sk@2], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([sr_customer_sk@6, sr_item_sk@5], 3), input_partitions=3 │ ProjectionExec: expr=[ss_sold_date_sk@4 as ss_sold_date_sk, ss_item_sk@5 as ss_item_sk, ss_store_sk@6 as ss_store_sk, ss_quantity@7 as ss_quantity, sr_returned_date_sk@0 as sr_returned_date_sk, sr_item_sk@1 as sr_item_sk, sr_customer_sk@2 as sr_customer_sk, sr_return_quantity@3 as sr_return_quantity] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_customer_sk@2, ss_customer_sk@2), (sr_item_sk@1, ss_item_sk@1), (sr_ticket_number@3, ss_ticket_number@4)], projection=[sr_returned_date_sk@0, sr_item_sk@1, sr_customer_sk@2, sr_return_quantity@4, ss_sold_date_sk@5, ss_item_sk@6, ss_store_sk@8, ss_quantity@10] - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([sr_customer_sk@2, sr_item_sk@1, sr_ticket_number@3], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([sr_customer_sk@2, sr_item_sk@1, sr_ticket_number@3], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_customer_sk@2, ss_item_sk@1, ss_ticket_number@4], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@2, ss_item_sk@1, ss_ticket_number@4], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); @@ -2283,9 +2283,9 @@ mod tests { │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_bill_customer_sk@2 as cs_bill_customer_sk, cs_item_sk@3 as cs_item_sk, cs_quantity@4 as cs_quantity, cs_list_price@5 as cs_list_price, cs_sales_price@6 as cs_sales_price, cs_coupon_amt@7 as cs_coupon_amt, cs_net_profit@8 as cs_net_profit, cd_dep_count@0 as cd_dep_count] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(cd1.cd_demo_sk AS Float64)@2, cs_bill_cdemo_sk@2)], projection=[cd_dep_count@1, cs_sold_date_sk@3, cs_bill_customer_sk@4, cs_item_sk@6, cs_quantity@7, cs_list_price@8, cs_sales_price@9, cs_coupon_amt@10, cs_net_profit@11] - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── @@ -2312,26 +2312,26 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk, c_birth_month, c_birth_year], file_type=parquet, predicate=c_birth_month@3 IN (SET) ([1, 6, 8, 9, 12, 2]) AND DynamicFilter [ empty ], pruning_predicate=c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 1 AND 1 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 6 AND 6 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 8 AND 8 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 9 AND 9 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 12 AND 12 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 2 AND 2 <= c_birth_month_max@1, required_guarantees=[c_birth_month in (1, 12, 2, 6, 8, 9)] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(cd1.cd_demo_sk AS Float64)@2], 3), input_partitions=3 + │ RepartitionExec: partitioning=Hash([CAST(cd1.cd_demo_sk AS Float64)@2], 3), input_partitions=2 │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, cd_dep_count@1 as cd_dep_count, CAST(cd_demo_sk@0 AS Float64) as CAST(cd1.cd_demo_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: cd_gender@1 = F AND cd_education_status@2 = Unknown, projection=[cd_demo_sk@0, cd_dep_count@3] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_education_status, cd_dep_count], file_type=parquet, predicate=cd_gender@1 = F AND cd_education_status@2 = Unknown, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= F AND F <= cd_gender_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Unknown AND Unknown <= cd_education_status_max@5, required_guarantees=[cd_education_status in (Unknown), cd_gender in (F)] └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_bill_cdemo_sk@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_bill_cdemo_sk@2], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_bill_cdemo_sk, cs_item_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(cd2.cd_demo_sk AS Float64)@1], 3), input_partitions=3 + │ RepartitionExec: partitioning=Hash([CAST(cd2.cd_demo_sk AS Float64)@1], 3), input_partitions=2 │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(cd2.cd_demo_sk AS Float64)] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk], file_type=parquet └────────────────────────────────────────────────── "); @@ -2583,7 +2583,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@3, cs_bill_customer_sk@1)], projection=[c_first_name@1, c_last_name@2, cs_sold_date_sk@4, cs_bill_customer_sk@5, cs_item_sk@6, cs_quantity@7, cs_list_price@8] │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[i_item_sk@0 as item_sk] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: count(Int64(1))@1 > 4, projection=[i_item_sk@0] @@ -2632,12 +2632,12 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@1, ss_customer_sk@1)], projection=[c_customer_sk@0, ss_sold_date_sk@2, ss_quantity@4, ss_sales_price@5] │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[ss_quantity@1 as ss_quantity, ss_sales_price@2 as ss_sales_price, c_customer_sk@0 as c_customer_sk] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@1, ss_customer_sk@0)], projection=[c_customer_sk@0, ss_quantity@3, ss_sales_price@4] │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, sum(web_sales.ws_quantity * web_sales.ws_list_price)@2 as sales] │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name], aggr=[sum(web_sales.ws_quantity * web_sales.ws_list_price)] │ CoalesceBatchesExec: target_batch_size=8192 @@ -2658,7 +2658,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@3, ws_bill_customer_sk@2)], projection=[c_first_name@1, c_last_name@2, ws_sold_date_sk@4, ws_item_sk@5, ws_bill_customer_sk@6, ws_quantity@7, ws_list_price@8] │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[i_item_sk@0 as item_sk] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: count(Int64(1))@1 > 4, projection=[i_item_sk@0] @@ -2707,12 +2707,12 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@1, ss_customer_sk@1)], projection=[c_customer_sk@0, ss_sold_date_sk@2, ss_quantity@4, ss_sales_price@5] │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[ss_quantity@1 as ss_quantity, ss_sales_price@2 as ss_sales_price, c_customer_sk@0 as c_customer_sk] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@1, ss_customer_sk@0)], projection=[c_customer_sk@0, ss_quantity@3, ss_sales_price@4] │ [Stage 17] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 18] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 18] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] @@ -2730,10 +2730,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -2760,10 +2760,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] @@ -2774,10 +2774,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_customer_sk@0], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@0], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -2796,10 +2796,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@2], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 13 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -2826,10 +2826,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 17 ── Tasks: t0:[p0..p2] t1:[p0..p2] @@ -2840,10 +2840,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 18 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 18 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_customer_sk@0], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@0], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── "); @@ -2884,8 +2884,8 @@ mod tests { │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_ticket_number@1, ss_ticket_number@3), (sr_item_sk@0, ss_item_sk@0)], projection=[ss_item_sk@2, ss_customer_sk@3, ss_store_sk@4, ss_net_paid@6] - │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: i_color@3 = peach │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 @@ -2920,8 +2920,8 @@ mod tests { │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_ticket_number@1, ss_ticket_number@3), (sr_item_sk@0, ss_item_sk@0)], projection=[ss_item_sk@2, ss_customer_sk@3, ss_store_sk@4, ss_net_paid@6] - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet, predicate=DynamicFilter [ empty ] │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_birth_country@4 as c_birth_country, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] @@ -2938,16 +2938,16 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_market_id, s_state, s_zip], file_type=parquet, predicate=s_market_id@2 = 8, pruning_predicate=s_market_id_null_count@2 != row_count@3 AND s_market_id_min@0 <= 8 AND 8 <= s_market_id_max@1, required_guarantees=[s_market_id in (8)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_ticket_number@3, ss_item_sk@0], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_ticket_number@3, ss_item_sk@0], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -2958,16 +2958,16 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_market_id, s_state, s_zip], file_type=parquet, predicate=s_market_id@2 = 8, pruning_predicate=s_market_id_null_count@2 != row_count@3 AND s_market_id_min@0 <= 8 AND 8 <= s_market_id_max@1, required_guarantees=[s_market_id in (8)] └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_ticket_number@3, ss_item_sk@0], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_ticket_number@3, ss_item_sk@0], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); @@ -3004,8 +3004,8 @@ mod tests { │ ProjectionExec: expr=[ss_sold_date_sk@2 as ss_sold_date_sk, ss_item_sk@3 as ss_item_sk, ss_store_sk@4 as ss_store_sk, ss_net_profit@5 as ss_net_profit, sr_returned_date_sk@6 as sr_returned_date_sk, sr_net_loss@7 as sr_net_loss, cs_sold_date_sk@0 as cs_sold_date_sk, cs_net_profit@1 as cs_net_profit] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cs_bill_customer_sk@1, sr_customer_sk@6), (cs_item_sk@2, sr_item_sk@5)], projection=[cs_sold_date_sk@0, cs_net_profit@3, ss_sold_date_sk@4, ss_item_sk@5, ss_store_sk@6, ss_net_profit@7, sr_returned_date_sk@8, sr_net_loss@11] - │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: d_moy@2 >= 4 AND d_moy@2 <= 10 AND d_year@1 = 2001, projection=[d_date_sk@0] @@ -3029,31 +3029,31 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 4 AND d_year@1 = 2001, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 4 AND 4 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_moy in (4), d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1, cs_item_sk@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1, cs_item_sk@2], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_net_profit], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([sr_customer_sk@6, sr_item_sk@5], 3), input_partitions=3 │ ProjectionExec: expr=[ss_sold_date_sk@4 as ss_sold_date_sk, ss_item_sk@5 as ss_item_sk, ss_store_sk@6 as ss_store_sk, ss_net_profit@7 as ss_net_profit, sr_returned_date_sk@0 as sr_returned_date_sk, sr_item_sk@1 as sr_item_sk, sr_customer_sk@2 as sr_customer_sk, sr_net_loss@3 as sr_net_loss] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_customer_sk@2, ss_customer_sk@2), (sr_item_sk@1, ss_item_sk@1), (sr_ticket_number@3, ss_ticket_number@4)], projection=[sr_returned_date_sk@0, sr_item_sk@1, sr_customer_sk@2, sr_net_loss@4, ss_sold_date_sk@5, ss_item_sk@6, ss_store_sk@8, ss_net_profit@10] - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([sr_customer_sk@2, sr_item_sk@1, sr_ticket_number@3], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([sr_customer_sk@2, sr_item_sk@1, sr_ticket_number@3], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_customer_sk@2, ss_item_sk@1, ss_ticket_number@4], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@2, ss_item_sk@1, ss_ticket_number@4], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); @@ -3084,8 +3084,8 @@ mod tests { │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, cs_bill_cdemo_sk@1)], projection=[cs_sold_date_sk@2, cs_item_sk@4, cs_promo_sk@5, cs_quantity@6, cs_list_price@7, cs_sales_price@8, cs_coupon_amt@9] - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── @@ -3105,19 +3105,19 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=3 + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=2 │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_bill_cdemo_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_bill_cdemo_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_cdemo_sk, cs_item_sk, cs_promo_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); @@ -3154,8 +3154,8 @@ mod tests { │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, ss_cdemo_sk@2)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_store_sk@5, ss_quantity@6, ss_list_price@7, ss_sales_price@8, ss_coupon_amt@9] - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] │ ProjectionExec: expr=[i_item_id@0 as i_item_id, NULL as s_state, 1 as g_state, avg(results.agg1)@1 as agg1, avg(results.agg2)@2 as agg2, avg(results.agg3)@3 as agg3, avg(results.agg4)@4 as agg4] @@ -3177,8 +3177,8 @@ mod tests { │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, ss_cdemo_sk@2)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_store_sk@5, ss_quantity@6, ss_list_price@7, ss_sales_price@8, ss_coupon_amt@9] - │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] │ ProjectionExec: expr=[NULL as i_item_id, NULL as s_state, 1 as g_state, avg(results.agg1)@0 as agg1, avg(results.agg2)@1 as agg2, avg(results.agg3)@2 as agg3, avg(results.agg4)@3 as agg4] @@ -3199,8 +3199,8 @@ mod tests { │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, ss_cdemo_sk@2)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_store_sk@5, ss_quantity@6, ss_list_price@7, ss_sales_price@8, ss_coupon_amt@9] - │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── @@ -3220,19 +3220,19 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=3 + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=2 │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -3251,19 +3251,19 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=3 + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=2 │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -3282,19 +3282,19 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] └────────────────────────────────────────────────── - ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=3 + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=2 │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] └────────────────────────────────────────────────── - ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); @@ -3311,75 +3311,75 @@ mod tests { │ ProjectionExec: expr=[avg(store_sales.ss_list_price)@0 as b6_lp, count(store_sales.ss_list_price)@1 as b6_cnt, count(DISTINCT store_sales.ss_list_price)@2 as b6_cntd] │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ ProjectionExec: expr=[b1_lp@3 as b1_lp, b1_cnt@4 as b1_cnt, b1_cntd@5 as b1_cntd, b2_lp@6 as b2_lp, b2_cnt@7 as b2_cnt, b2_cntd@8 as b2_cntd, b3_lp@9 as b3_lp, b3_cnt@10 as b3_cnt, b3_cntd@11 as b3_cntd, b4_lp@12 as b4_lp, b4_cnt@13 as b4_cnt, b4_cntd@14 as b4_cntd, b5_lp@0 as b5_lp, b5_cnt@1 as b5_cnt, b5_cntd@2 as b5_cntd] │ CrossJoinExec │ ProjectionExec: expr=[avg(store_sales.ss_list_price)@0 as b5_lp, count(store_sales.ss_list_price)@1 as b5_cnt, count(DISTINCT store_sales.ss_list_price)@2 as b5_cntd] │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ ProjectionExec: expr=[b1_lp@3 as b1_lp, b1_cnt@4 as b1_cnt, b1_cntd@5 as b1_cntd, b2_lp@6 as b2_lp, b2_cnt@7 as b2_cnt, b2_cntd@8 as b2_cntd, b3_lp@9 as b3_lp, b3_cnt@10 as b3_cnt, b3_cntd@11 as b3_cntd, b4_lp@0 as b4_lp, b4_cnt@1 as b4_cnt, b4_cntd@2 as b4_cntd] │ CrossJoinExec │ ProjectionExec: expr=[avg(store_sales.ss_list_price)@0 as b4_lp, count(store_sales.ss_list_price)@1 as b4_cnt, count(DISTINCT store_sales.ss_list_price)@2 as b4_cntd] │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] │ CoalescePartitionsExec - │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ ProjectionExec: expr=[b1_lp@3 as b1_lp, b1_cnt@4 as b1_cnt, b1_cntd@5 as b1_cntd, b2_lp@6 as b2_lp, b2_cnt@7 as b2_cnt, b2_cntd@8 as b2_cntd, b3_lp@0 as b3_lp, b3_cnt@1 as b3_cnt, b3_cntd@2 as b3_cntd] │ CrossJoinExec │ ProjectionExec: expr=[avg(store_sales.ss_list_price)@0 as b3_lp, count(store_sales.ss_list_price)@1 as b3_cnt, count(DISTINCT store_sales.ss_list_price)@2 as b3_cntd] │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] │ CoalescePartitionsExec - │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ CrossJoinExec │ ProjectionExec: expr=[avg(store_sales.ss_list_price)@0 as b1_lp, count(store_sales.ss_list_price)@1 as b1_cnt, count(DISTINCT store_sales.ss_list_price)@2 as b1_cntd] │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] │ CoalescePartitionsExec - │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ ProjectionExec: expr=[avg(store_sales.ss_list_price)@0 as b2_lp, count(store_sales.ss_list_price)@1 as b2_cnt, count(DISTINCT store_sales.ss_list_price)@2 as b2_cntd] │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] │ CoalescePartitionsExec - │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 26 AND ss_quantity@0 <= 30 AND (ss_list_price@2 >= Some(15400),5,2 AND ss_list_price@2 <= Some(16400),5,2 OR ss_coupon_amt@3 >= Some(732600),7,2 AND ss_coupon_amt@3 <= Some(832600),7,2 OR ss_wholesale_cost@1 >= Some(700),5,2 AND ss_wholesale_cost@1 <= Some(2700),5,2), projection=[ss_list_price@2] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@0 >= 26 AND ss_quantity@0 <= 30 AND (ss_list_price@2 >= Some(15400),5,2 AND ss_list_price@2 <= Some(16400),5,2 OR ss_coupon_amt@3 >= Some(732600),7,2 AND ss_coupon_amt@3 <= Some(832600),7,2 OR ss_wholesale_cost@1 >= Some(700),5,2 AND ss_wholesale_cost@1 <= Some(2700),5,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 26 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 30 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(15400),5,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(16400),5,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(732600),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(832600),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(700),5,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(2700),5,2), required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 21 AND ss_quantity@0 <= 25 AND (ss_list_price@2 >= Some(12200),5,2 AND ss_list_price@2 <= Some(13200),5,2 OR ss_coupon_amt@3 >= Some(83600),7,2 AND ss_coupon_amt@3 <= Some(183600),7,2 OR ss_wholesale_cost@1 >= Some(1700),5,2 AND ss_wholesale_cost@1 <= Some(3700),5,2), projection=[ss_list_price@2] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@0 >= 21 AND ss_quantity@0 <= 25 AND (ss_list_price@2 >= Some(12200),5,2 AND ss_list_price@2 <= Some(13200),5,2 OR ss_coupon_amt@3 >= Some(83600),7,2 AND ss_coupon_amt@3 <= Some(183600),7,2 OR ss_wholesale_cost@1 >= Some(1700),5,2 AND ss_wholesale_cost@1 <= Some(3700),5,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 21 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 25 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(12200),5,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(13200),5,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(83600),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(183600),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(1700),5,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(3700),5,2), required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 3 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 16 AND ss_quantity@0 <= 20 AND (ss_list_price@2 >= Some(13500),5,2 AND ss_list_price@2 <= Some(14500),5,2 OR ss_coupon_amt@3 >= Some(607100),7,2 AND ss_coupon_amt@3 <= Some(707100),7,2 OR ss_wholesale_cost@1 >= Some(3800),5,2 AND ss_wholesale_cost@1 <= Some(5800),5,2), projection=[ss_list_price@2] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@0 >= 16 AND ss_quantity@0 <= 20 AND (ss_list_price@2 >= Some(13500),5,2 AND ss_list_price@2 <= Some(14500),5,2 OR ss_coupon_amt@3 >= Some(607100),7,2 AND ss_coupon_amt@3 <= Some(707100),7,2 OR ss_wholesale_cost@1 >= Some(3800),5,2 AND ss_wholesale_cost@1 <= Some(5800),5,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 16 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 20 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(13500),5,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(14500),5,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(607100),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(707100),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(3800),5,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(5800),5,2), required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 4 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 11 AND ss_quantity@0 <= 15 AND (ss_list_price@2 >= Some(14200),5,2 AND ss_list_price@2 <= Some(15200),5,2 OR ss_coupon_amt@3 >= Some(1221400),7,2 AND ss_coupon_amt@3 <= Some(1321400),7,2 OR ss_wholesale_cost@1 >= Some(7900),5,2 AND ss_wholesale_cost@1 <= Some(9900),5,2), projection=[ss_list_price@2] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@0 >= 11 AND ss_quantity@0 <= 15 AND (ss_list_price@2 >= Some(14200),5,2 AND ss_list_price@2 <= Some(15200),5,2 OR ss_coupon_amt@3 >= Some(1221400),7,2 AND ss_coupon_amt@3 <= Some(1321400),7,2 OR ss_wholesale_cost@1 >= Some(7900),5,2 AND ss_wholesale_cost@1 <= Some(9900),5,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 11 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 15 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(14200),5,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(15200),5,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(1221400),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(1321400),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(7900),5,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(9900),5,2), required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 5 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 0 AND ss_quantity@0 <= 5 AND (ss_list_price@2 >= Some(800),5,2 AND ss_list_price@2 <= Some(1800),5,2 OR ss_coupon_amt@3 >= Some(45900),7,2 AND ss_coupon_amt@3 <= Some(145900),7,2 OR ss_wholesale_cost@1 >= Some(5700),5,2 AND ss_wholesale_cost@1 <= Some(7700),5,2), projection=[ss_list_price@2] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@0 >= 0 AND ss_quantity@0 <= 5 AND (ss_list_price@2 >= Some(800),5,2 AND ss_list_price@2 <= Some(1800),5,2 OR ss_coupon_amt@3 >= Some(45900),7,2 AND ss_coupon_amt@3 <= Some(145900),7,2 OR ss_wholesale_cost@1 >= Some(5700),5,2 AND ss_wholesale_cost@1 <= Some(7700),5,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 0 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 5 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(800),5,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(1800),5,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(45900),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(145900),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(5700),5,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(7700),5,2), required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 6 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_quantity@0 >= 6 AND ss_quantity@0 <= 10 AND (ss_list_price@2 >= Some(9000),5,2 AND ss_list_price@2 <= Some(10000),5,2 OR ss_coupon_amt@3 >= Some(232300),7,2 AND ss_coupon_amt@3 <= Some(332300),7,2 OR ss_wholesale_cost@1 >= Some(3100),5,2 AND ss_wholesale_cost@1 <= Some(5100),5,2), projection=[ss_list_price@2] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@0 >= 6 AND ss_quantity@0 <= 10 AND (ss_list_price@2 >= Some(9000),5,2 AND ss_list_price@2 <= Some(10000),5,2 OR ss_coupon_amt@3 >= Some(232300),7,2 AND ss_coupon_amt@3 <= Some(332300),7,2 OR ss_wholesale_cost@1 >= Some(3100),5,2 AND ss_wholesale_cost@1 <= Some(5100),5,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 6 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 10 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(9000),5,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(10000),5,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(232300),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(332300),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(3100),5,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(5100),5,2), required_guarantees=[] └────────────────────────────────────────────────── "); @@ -3417,8 +3417,8 @@ mod tests { │ ProjectionExec: expr=[ss_sold_date_sk@2 as ss_sold_date_sk, ss_item_sk@3 as ss_item_sk, ss_store_sk@4 as ss_store_sk, ss_quantity@5 as ss_quantity, sr_returned_date_sk@6 as sr_returned_date_sk, sr_return_quantity@7 as sr_return_quantity, cs_sold_date_sk@0 as cs_sold_date_sk, cs_quantity@1 as cs_quantity] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cs_bill_customer_sk@1, sr_customer_sk@6), (cs_item_sk@2, sr_item_sk@5)], projection=[cs_sold_date_sk@0, cs_quantity@3, ss_sold_date_sk@4, ss_item_sk@5, ss_store_sk@6, ss_quantity@7, sr_returned_date_sk@8, sr_return_quantity@11] - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001, projection=[d_date_sk@0] @@ -3445,31 +3445,31 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@2 = 9 AND d_year@1 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 9 AND 9 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (9), d_year in (1999)] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1, cs_item_sk@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1, cs_item_sk@2], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([sr_customer_sk@6, sr_item_sk@5], 3), input_partitions=3 │ ProjectionExec: expr=[ss_sold_date_sk@4 as ss_sold_date_sk, ss_item_sk@5 as ss_item_sk, ss_store_sk@6 as ss_store_sk, ss_quantity@7 as ss_quantity, sr_returned_date_sk@0 as sr_returned_date_sk, sr_item_sk@1 as sr_item_sk, sr_customer_sk@2 as sr_customer_sk, sr_return_quantity@3 as sr_return_quantity] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_customer_sk@2, ss_customer_sk@2), (sr_item_sk@1, ss_item_sk@1), (sr_ticket_number@3, ss_ticket_number@4)], projection=[sr_returned_date_sk@0, sr_item_sk@1, sr_customer_sk@2, sr_return_quantity@4, ss_sold_date_sk@5, ss_item_sk@6, ss_store_sk@8, ss_quantity@10] - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([sr_customer_sk@2, sr_item_sk@1, sr_ticket_number@3], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([sr_customer_sk@2, sr_item_sk@1, sr_ticket_number@3], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_customer_sk@2, ss_item_sk@1, ss_ticket_number@4], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@2, ss_item_sk@1, ss_ticket_number@4], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); @@ -4177,7 +4177,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(inv_item_sk@1, i_item_sk@0)], projection=[inv_date_sk@0, i_item_sk@2, i_item_id@3, i_item_desc@4, i_current_price@5] │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: i_current_price@3 >= Some(6800),4,2 AND i_current_price@3 <= Some(9800),4,2 AND i_manufact_id@4 IN (SET) ([677, 940, 694, 808]), projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 @@ -4188,10 +4188,10 @@ mod tests { │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-02-01 AND d_date@1 <= 2000-04-01 AND DynamicFilter [ empty ], pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-01, required_guarantees=[] │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: inv_quantity_on_hand@2 >= 100 AND inv_quantity_on_hand@2 <= 500, projection=[inv_date_sk@0, inv_item_sk@1] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], ...]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@2 >= 100 AND inv_quantity_on_hand@2 <= 500, pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] └────────────────────────────────────────────────── "); @@ -4329,7 +4329,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, inv_item_sk@1)], projection=[i_item_sk@0, inv_date_sk@1, inv_warehouse_sk@3, inv_quantity_on_hand@4] │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([i_item_sk@1, w_warehouse_sk@0], 3), input_partitions=3 │ ProjectionExec: expr=[w_warehouse_sk@0 as w_warehouse_sk, i_item_sk@1 as i_item_sk, d_moy@2 as d_moy, avg(inventory.inv_quantity_on_hand)@4 as mean, CASE avg(inventory.inv_quantity_on_hand)@4 WHEN 0 THEN NULL ELSE stddev(inventory.inv_quantity_on_hand)@3 / avg(inventory.inv_quantity_on_hand)@4 END as cov] @@ -4354,7 +4354,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, inv_item_sk@1)], projection=[i_item_sk@0, inv_date_sk@1, inv_warehouse_sk@3, inv_quantity_on_hand@4] │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ CoalesceBatchesExec: target_batch_size=8192 @@ -4370,10 +4370,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([inv_item_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([inv_item_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], ...]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -4390,10 +4390,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([inv_item_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([inv_item_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], ...]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); @@ -4426,8 +4426,8 @@ mod tests { │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_warehouse_sk@2 as cs_warehouse_sk, cs_item_sk@3 as cs_item_sk, cs_sales_price@4 as cs_sales_price, cr_refunded_cash@0 as cr_refunded_cash] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(cr_order_number@1, cs_order_number@3), (cr_item_sk@0, cs_item_sk@2)], projection=[cr_refunded_cash@2, cs_sold_date_sk@3, cs_warehouse_sk@4, cs_item_sk@5, cs_sales_price@7] - │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: i_current_price@2 >= Some(99),4,2 AND i_current_price@2 <= Some(149),4,2, projection=[i_item_sk@0, i_item_id@1] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 @@ -4443,16 +4443,16 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet]]}, projection=[w_warehouse_sk, w_state], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cr_order_number@1, cr_item_sk@0], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cr_order_number@1, cr_item_sk@0], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_refunded_cash], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_order_number@3, cs_item_sk@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_order_number@3, cs_item_sk@2], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_warehouse_sk, cs_item_sk, cs_order_number, cs_sales_price], file_type=parquet └────────────────────────────────────────────────── "#); @@ -4619,10 +4619,10 @@ mod tests { │ NestedLoopJoinExec: join_type=Left │ CoalescePartitionsExec │ AggregateExec: mode=FinalPartitioned, gby=[ss_item_sk@0 as ss_item_sk], aggr=[avg(ss1.ss_net_profit)] - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[CAST(avg(store_sales.ss_net_profit)@1 AS Float64) as rank_col] │ AggregateExec: mode=FinalPartitioned, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(store_sales.ss_net_profit)], ordering_mode=Sorted - │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([rnk@1], 3), input_partitions=3 │ ProjectionExec: expr=[item_sk@0 as item_sk, rank() ORDER BY [v2.rank_col DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 as rnk] @@ -4639,45 +4639,45 @@ mod tests { │ NestedLoopJoinExec: join_type=Left │ CoalescePartitionsExec │ AggregateExec: mode=FinalPartitioned, gby=[ss_item_sk@0 as ss_item_sk], aggr=[avg(ss1.ss_net_profit)] - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[CAST(avg(store_sales.ss_net_profit)@1 AS Float64) as rank_col] │ AggregateExec: mode=FinalPartitioned, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(store_sales.ss_net_profit)], ordering_mode=Sorted - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=Hash([ss_item_sk@0], 3), input_partitions=2 │ AggregateExec: mode=Partial, gby=[ss_item_sk@0 as ss_item_sk], aggr=[avg(ss1.ss_net_profit)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_store_sk@1 = 4, projection=[ss_item_sk@0, ss_net_profit@2] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_item_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=ss_store_sk@1 = 4, pruning_predicate=ss_store_sk_null_count@2 != row_count@3 AND ss_store_sk_min@0 <= 4 AND 4 <= ss_store_sk_max@1, required_guarantees=[ss_store_sk in (4)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_store_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=Hash([ss_store_sk@0], 3), input_partitions=2 │ AggregateExec: mode=Partial, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(store_sales.ss_net_profit)], ordering_mode=Sorted │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_store_sk@1 = 4 AND ss_addr_sk@0 IS NULL, projection=[ss_store_sk@1, ss_net_profit@2] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_addr_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=ss_store_sk@1 = 4 AND ss_addr_sk@0 IS NULL, pruning_predicate=ss_store_sk_null_count@2 != row_count@3 AND ss_store_sk_min@0 <= 4 AND 4 <= ss_store_sk_max@1 AND ss_addr_sk_null_count@4 > 0, required_guarantees=[ss_store_sk in (4)] └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_item_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=Hash([ss_item_sk@0], 3), input_partitions=2 │ AggregateExec: mode=Partial, gby=[ss_item_sk@0 as ss_item_sk], aggr=[avg(ss1.ss_net_profit)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_store_sk@1 = 4, projection=[ss_item_sk@0, ss_net_profit@2] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_item_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=ss_store_sk@1 = 4, pruning_predicate=ss_store_sk_null_count@2 != row_count@3 AND ss_store_sk_min@0 <= 4 AND 4 <= ss_store_sk_max@1, required_guarantees=[ss_store_sk in (4)] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_store_sk@0], 3), input_partitions=3 + │ RepartitionExec: partitioning=Hash([ss_store_sk@0], 3), input_partitions=2 │ AggregateExec: mode=Partial, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(store_sales.ss_net_profit)], ordering_mode=Sorted │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_store_sk@1 = 4 AND ss_addr_sk@0 IS NULL, projection=[ss_store_sk@1, ss_net_profit@2] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_addr_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=ss_store_sk@1 = 4 AND ss_addr_sk@0 IS NULL, pruning_predicate=ss_store_sk_null_count@2 != row_count@3 AND ss_store_sk_min@0 <= 4 AND 4 <= ss_store_sk_max@1 AND ss_addr_sk_null_count@4 > 0, required_guarantees=[ss_store_sk in (4)] └────────────────────────────────────────────────── ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] @@ -4724,7 +4724,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@3)], projection=[ca_city@1, ca_zip@2, ws_sold_date_sk@3, ws_item_sk@4, ws_sales_price@5] │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] │ CoalesceBatchesExec: target_batch_size=8192 @@ -4747,27 +4747,27 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_city, ca_zip], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([c_current_addr_sk@3], 3), input_partitions=3 │ ProjectionExec: expr=[ws_sold_date_sk@1 as ws_sold_date_sk, ws_item_sk@2 as ws_item_sk, ws_sales_price@3 as ws_sales_price, c_current_addr_sk@0 as c_current_addr_sk] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@2, ws_bill_customer_sk@2)], projection=[c_current_addr_sk@1, ws_sold_date_sk@3, ws_item_sk@4, ws_sales_price@6] │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@2], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@2], 9), input_partitions=3 │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@2], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@2], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); @@ -4887,7 +4887,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand@1, i_category@2, ss_sold_date_sk@3, ss_store_sk@5, ss_sales_price@6] │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, sum(store_sales.ss_sales_price)@6 as sum_sales, rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 AS Decimal128(20, 0)) + Some(1),20,0 as v1_lag.rn + Decimal128(Some(1),20,0)] │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_year@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST], preserve_partitioning=[true] @@ -4910,7 +4910,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand@1, i_category@2, ss_sold_date_sk@3, ss_store_sk@5, ss_sales_price@6] │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, sum(store_sales.ss_sales_price)@6 as sum_sales, rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 AS Decimal128(20, 0)) - Some(1),20,0 as v1_lead.rn - Decimal128(Some(1),20,0)] │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_year@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST], preserve_partitioning=[true] @@ -4933,7 +4933,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand@1, i_category@2, ss_sold_date_sk@3, ss_store_sk@5, ss_sales_price@6] │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_name@1 as s_store_name, s_company_name@2 as s_company_name, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] @@ -4955,10 +4955,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 5 ── Tasks: t0:[p0..p1] t1:[p2..p3] @@ -4981,10 +4981,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 9 ── Tasks: t0:[p0..p1] t1:[p2..p3] @@ -5007,10 +5007,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "#); @@ -5102,7 +5102,7 @@ mod tests { │ FilterExec: wr_return_amt@5 > Some(1000000),7,2 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(ws_order_number@2, wr_order_number@1), (ws_item_sk@1, wr_item_sk@0)], projection=[ws_sold_date_sk@0, ws_item_sk@1, ws_quantity@3, ws_net_paid@4, wr_return_quantity@7, wr_return_amt@8] - │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ ProjectionExec: expr=[catalog as channel, item@0 as item, return_ratio@1 as return_ratio, return_rank@2 as return_rank, currency_rank@3 as currency_rank] │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as return_rank, rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as currency_rank] @@ -5128,8 +5128,8 @@ mod tests { │ FilterExec: cr_return_amount@5 > Some(1000000),7,2 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(cs_order_number@2, cr_order_number@1), (cs_item_sk@1, cr_item_sk@0)], projection=[cs_sold_date_sk@0, cs_item_sk@1, cs_quantity@3, cs_net_paid@4, cr_return_quantity@7, cr_return_amount@8] - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[store as channel, item@0 as item, return_ratio@1 as return_ratio, return_rank@2 as return_rank, currency_rank@3 as currency_rank] │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as return_rank, rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as currency_rank] │ CoalesceBatchesExec: target_batch_size=8192 @@ -5155,8 +5155,8 @@ mod tests { │ ProjectionExec: expr=[ss_sold_date_sk@2 as ss_sold_date_sk, ss_item_sk@3 as ss_item_sk, ss_quantity@4 as ss_quantity, ss_net_paid@5 as ss_net_paid, sr_return_quantity@0 as sr_return_quantity, sr_return_amt@1 as sr_return_amt] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(sr_ticket_number@1, ss_ticket_number@2), (sr_item_sk@0, ss_item_sk@1)], projection=[sr_return_quantity@2, sr_return_amt@3, ss_sold_date_sk@4, ss_item_sk@5, ss_quantity@7, ss_net_paid@8] - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] @@ -5166,12 +5166,12 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_order_number@2, ws_item_sk@1], 3), input_partitions=3 + │ RepartitionExec: partitioning=Hash([ws_order_number@2, ws_item_sk@1], 3), input_partitions=2 │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ws_net_profit@5 > Some(100),7,2 AND ws_net_paid@4 > Some(0),7,2 AND ws_quantity@3 > 0, projection=[ws_sold_date_sk@0, ws_item_sk@1, ws_order_number@2, ws_quantity@3, ws_net_paid@4] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_net_paid, ws_net_profit], file_type=parquet, predicate=ws_net_profit@5 > Some(100),7,2 AND ws_net_paid@4 > Some(0),7,2 AND ws_quantity@3 > 0, pruning_predicate=ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 > Some(100),7,2 AND ws_net_paid_null_count@4 != row_count@2 AND ws_net_paid_max@3 > Some(0),7,2 AND ws_quantity_null_count@6 != row_count@2 AND ws_quantity_max@5 > 0, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] @@ -5189,18 +5189,18 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_order_number@2, cs_item_sk@1], 3), input_partitions=3 + │ RepartitionExec: partitioning=Hash([cs_order_number@2, cs_item_sk@1], 3), input_partitions=2 │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: cs_net_profit@5 > Some(100),7,2 AND cs_net_paid@4 > Some(0),7,2 AND cs_quantity@3 > 0, projection=[cs_sold_date_sk@0, cs_item_sk@1, cs_order_number@2, cs_quantity@3, cs_net_paid@4] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_net_paid, cs_net_profit], file_type=parquet, predicate=cs_net_profit@5 > Some(100),7,2 AND cs_net_paid@4 > Some(0),7,2 AND cs_quantity@3 > 0, pruning_predicate=cs_net_profit_null_count@1 != row_count@2 AND cs_net_profit_max@0 > Some(100),7,2 AND cs_net_paid_null_count@4 != row_count@2 AND cs_net_paid_max@3 > Some(0),7,2 AND cs_quantity_null_count@6 != row_count@2 AND cs_quantity_max@5 > 0, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cr_order_number@1, cr_item_sk@0], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cr_order_number@1, cr_item_sk@0], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -5211,18 +5211,18 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@1 = 2001 AND d_moy@2 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_ticket_number@2, ss_item_sk@1], 3), input_partitions=3 + │ RepartitionExec: partitioning=Hash([ss_ticket_number@2, ss_item_sk@1], 3), input_partitions=2 │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_net_profit@5 > Some(100),6,2 AND ss_net_paid@4 > Some(0),7,2 AND ss_quantity@3 > 0, projection=[ss_sold_date_sk@0, ss_item_sk@1, ss_ticket_number@2, ss_quantity@3, ss_net_paid@4] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_net_paid, ss_net_profit], file_type=parquet, predicate=ss_net_profit@5 > Some(100),6,2 AND ss_net_paid@4 > Some(0),7,2 AND ss_quantity@3 > 0, pruning_predicate=ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 > Some(100),6,2 AND ss_net_paid_null_count@4 != row_count@2 AND ss_net_paid_max@3 > Some(0),7,2 AND ss_quantity_null_count@6 != row_count@2 AND ss_quantity_max@5 > 0, required_guarantees=[] └────────────────────────────────────────────────── "#); @@ -5255,8 +5255,8 @@ mod tests { │ ProjectionExec: expr=[ss_sold_date_sk@1 as ss_sold_date_sk, ss_store_sk@2 as ss_store_sk, sr_returned_date_sk@0 as sr_returned_date_sk] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_ticket_number@3, ss_ticket_number@4), (sr_item_sk@1, ss_item_sk@1), (sr_customer_sk@2, ss_customer_sk@2)], projection=[sr_returned_date_sk@0, ss_sold_date_sk@4, ss_store_sk@7] - │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk], file_type=parquet @@ -5271,16 +5271,16 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([sr_ticket_number@3, sr_item_sk@1, sr_customer_sk@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([sr_ticket_number@3, sr_item_sk@1, sr_customer_sk@2], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_ticket_number@4, ss_item_sk@1, ss_customer_sk@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_ticket_number@4, ss_item_sk@1, ss_customer_sk@2], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); @@ -5777,7 +5777,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@2)], projection=[i_brand@1, i_category@2, cs_sold_date_sk@3, cs_call_center_sk@4, cs_sales_price@6] │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, sum(catalog_sales.cs_sales_price)@5 as sum_sales, rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 AS Decimal128(20, 0)) + Some(1),20,0 as v1_lag.rn + Decimal128(Some(1),20,0)] │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_year@3 ASC NULLS LAST, d_moy@4 ASC NULLS LAST], preserve_partitioning=[true] @@ -5800,7 +5800,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@2)], projection=[i_brand@1, i_category@2, cs_sold_date_sk@3, cs_call_center_sk@4, cs_sales_price@6] │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, sum(catalog_sales.cs_sales_price)@5 as sum_sales, rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 AS Decimal128(20, 0)) - Some(1),20,0 as v1_lead.rn - Decimal128(Some(1),20,0)] │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_year@3 ASC NULLS LAST, d_moy@4 ASC NULLS LAST], preserve_partitioning=[true] @@ -5823,7 +5823,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@2)], projection=[i_brand@1, i_category@2, cs_sold_date_sk@3, cs_call_center_sk@4, cs_sales_price@6] │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] │ ProjectionExec: expr=[cc_call_center_sk@0 as cc_call_center_sk, cc_name@1 as cc_name, CAST(cc_call_center_sk@0 AS Float64) as CAST(call_center.cc_call_center_sk AS Float64)] @@ -5845,10 +5845,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_item_sk@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_item_sk@2], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 5 ── Tasks: t0:[p0..p1] t1:[p2..p3] @@ -5871,10 +5871,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_item_sk@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_item_sk@2], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 9 ── Tasks: t0:[p0..p1] t1:[p2..p3] @@ -5897,10 +5897,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_item_sk@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_item_sk@2], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "#); @@ -5926,7 +5926,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(d_date@2, d_date@0)], projection=[ws_ext_sales_price@0, i_item_id@1] │ CoalescePartitionsExec - │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 5] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@1] │ CoalescePartitionsExec @@ -5945,7 +5945,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(d_date@2, d_date@0)], projection=[cs_ext_sales_price@0, i_item_id@1] │ CoalescePartitionsExec - │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 11] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@1] │ CoalescePartitionsExec @@ -5960,7 +5960,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(d_date@2, d_date@0)], projection=[ss_ext_sales_price@0, i_item_id@1] │ CoalescePartitionsExec - │ [Stage 17] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 17] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@1] │ CoalescePartitionsExec @@ -5968,41 +5968,41 @@ mod tests { │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ ProjectionExec: expr=[ws_ext_sales_price@1 as ws_ext_sales_price, i_item_id@2 as i_item_id, d_date@0 as d_date] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@0)], projection=[d_date@1, ws_ext_sales_price@4, i_item_id@5] │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@2], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@2], 9), input_partitions=3 │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_sold_date_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([ws_sold_date_sk@0], 9), input_partitions=3 │ ProjectionExec: expr=[ws_sold_date_sk@1 as ws_sold_date_sk, ws_ext_sales_price@2 as ws_ext_sales_price, i_item_id@0 as i_item_id] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_item_id@1, ws_sold_date_sk@2, ws_ext_sales_price@4] │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 9), input_partitions=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -6012,41 +6012,41 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@0 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] └────────────────────────────────────────────────── - ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ ProjectionExec: expr=[cs_ext_sales_price@1 as cs_ext_sales_price, i_item_id@2 as i_item_id, d_date@0 as d_date] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@0)], projection=[d_date@1, cs_ext_sales_price@4, i_item_id@5] │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@2], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@2], 9), input_partitions=3 │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 10 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_sold_date_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([cs_sold_date_sk@0], 9), input_partitions=3 │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_ext_sales_price@2 as cs_ext_sales_price, i_item_id@0 as i_item_id] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_item_id@1, cs_sold_date_sk@2, cs_ext_sales_price@4] │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 8 ── Tasks: t0:[p0..p8] t1:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 9), input_partitions=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -6056,41 +6056,41 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@0 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] └────────────────────────────────────────────────── - ┌───── Stage 17 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 17 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ ProjectionExec: expr=[ss_ext_sales_price@1 as ss_ext_sales_price, i_item_id@2 as i_item_id, d_date@0 as d_date] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_date@1, ss_ext_sales_price@4, i_item_id@5] │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 13 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 13 ── Tasks: t0:[p0..p8] t1:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@2], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@2], 9), input_partitions=3 │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 16 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 16 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_sold_date_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([ss_sold_date_sk@0], 9), input_partitions=3 │ ProjectionExec: expr=[ss_sold_date_sk@1 as ss_sold_date_sk, ss_ext_sales_price@2 as ss_ext_sales_price, i_item_id@0 as i_item_id] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_item_id@1, ss_sold_date_sk@2, ss_ext_sales_price@4] │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 14 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 14 ── Tasks: t0:[p0..p8] t1:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 9), input_partitions=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 15 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 15 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 18 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -6125,7 +6125,7 @@ mod tests { │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq, ss_store_sk@1 as ss_store_sk, sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END)@2 as sun_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END)@3 as mon_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END)@4 as tue_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END)@5 as wed_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END)@6 as thu_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END)@7 as fri_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)@8 as sat_sales] │ AggregateExec: mode=FinalPartitioned, gby=[d_week_seq@0 as d_week_seq, ss_store_sk@1 as ss_store_sk], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: d_month_seq@0 >= 1212 AND d_month_seq@0 <= 1223, projection=[d_week_seq@1] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 @@ -6141,7 +6141,7 @@ mod tests { │ [Stage 5] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq, ss_store_sk@1 as ss_store_sk, sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END)@2 as sun_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END)@3 as mon_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END)@4 as tue_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END)@5 as wed_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END)@6 as thu_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END)@7 as fri_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)@8 as sat_sales] │ AggregateExec: mode=FinalPartitioned, gby=[d_week_seq@0 as d_week_seq, ss_store_sk@1 as ss_store_sk], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: d_month_seq@0 >= 1224 AND d_month_seq@0 <= 1235, projection=[d_week_seq@1] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 @@ -6152,7 +6152,7 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id, s_store_name], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([d_week_seq@0, ss_store_sk@1], 3), input_partitions=3 │ AggregateExec: mode=Partial, gby=[d_week_seq@2 as d_week_seq, ss_store_sk@0 as ss_store_sk], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] @@ -6160,20 +6160,20 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@0)], projection=[d_week_seq@1, d_day_name@2, ss_store_sk@5, ss_sales_price@6] │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 9), input_partitions=3 │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_week_seq@1 as d_week_seq, d_day_name@2 as d_day_name, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_sold_date_sk@0], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_sold_date_sk@0], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 5 ── Tasks: t0:[p0..p1] t1:[p2..p3] @@ -6181,7 +6181,7 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([d_week_seq@0, ss_store_sk@1], 3), input_partitions=3 │ AggregateExec: mode=Partial, gby=[d_week_seq@2 as d_week_seq, ss_store_sk@0 as ss_store_sk], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] @@ -6189,20 +6189,20 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@0)], projection=[d_week_seq@1, d_day_name@2, ss_store_sk@5, ss_sales_price@6] │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 9), input_partitions=3 │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_week_seq@1 as d_week_seq, d_day_name@2 as d_day_name, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_sold_date_sk@0], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_sold_date_sk@0], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── "#); @@ -6621,14 +6621,11 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, ss_item_sk@1)], projection=[ss_sold_date_sk@1, ss_item_sk@2, ss_customer_sk@3, ss_cdemo_sk@4, ss_hdemo_sk@5, ss_addr_sk@6, ss_store_sk@7, ss_promo_sk@8, ss_wholesale_cost@9, ss_list_price@10, ss_coupon_amt@11] │ CoalescePartitionsExec - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: CAST(sum(catalog_sales.cs_ext_list_price)@1 AS Decimal128(38, 2)) > Some(2),20,0 * sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)@2, projection=[cs_item_sk@0] - │ AggregateExec: mode=FinalPartitioned, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(catalog_sales.cs_ext_list_price), sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)] - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_item_sk@0, ss_item_sk@1), (sr_ticket_number@1, ss_ticket_number@8)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_customer_sk@4, ss_cdemo_sk@5, ss_hdemo_sk@6, ss_addr_sk@7, ss_store_sk@8, ss_promo_sk@9, ss_wholesale_cost@11, ss_list_price@12, ss_coupon_amt@13] - │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_cdemo_sk@1 as c_current_cdemo_sk, c_current_hdemo_sk@2 as c_current_hdemo_sk, c_current_addr_sk@3 as c_current_addr_sk, c_first_shipto_date_sk@4 as c_first_shipto_date_sk, c_first_sales_date_sk@5 as c_first_sales_date_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet @@ -6687,7 +6684,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(promotion.p_promo_sk AS Float64)@1, ss_promo_sk@3)], projection=[ss_item_sk@2, ss_hdemo_sk@3, ss_addr_sk@4, ss_wholesale_cost@6, ss_list_price@7, ss_coupon_amt@8, d_year@9, s_store_name@10, s_zip@11, c_current_hdemo_sk@12, c_current_addr_sk@13, d_year@14, d_year@15] │ CoalescePartitionsExec - │ [Stage 9] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ [Stage 10] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@10, CAST(cd2.cd_demo_sk AS Float64)@2)], filter=cd_marital_status@1 != cd_marital_status@0, projection=[ss_item_sk@0, ss_hdemo_sk@1, ss_addr_sk@2, ss_promo_sk@3, ss_wholesale_cost@4, ss_list_price@5, ss_coupon_amt@6, d_year@7, s_store_name@8, s_zip@9, c_current_hdemo_sk@11, c_current_addr_sk@12, d_year@13, d_year@14] │ CoalescePartitionsExec @@ -6707,23 +6704,20 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@3, ss_store_sk@5)], projection=[s_store_name@1, s_zip@2, ss_item_sk@4, ss_customer_sk@5, ss_cdemo_sk@6, ss_hdemo_sk@7, ss_addr_sk@8, ss_promo_sk@10, ss_wholesale_cost@11, ss_list_price@12, ss_coupon_amt@13, d_year@14] │ CoalescePartitionsExec - │ [Stage 10] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + │ [Stage 11] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_customer_sk@2 as ss_customer_sk, ss_cdemo_sk@3 as ss_cdemo_sk, ss_hdemo_sk@4 as ss_hdemo_sk, ss_addr_sk@5 as ss_addr_sk, ss_store_sk@6 as ss_store_sk, ss_promo_sk@7 as ss_promo_sk, ss_wholesale_cost@8 as ss_wholesale_cost, ss_list_price@9 as ss_list_price, ss_coupon_amt@10 as ss_coupon_amt, d_year@0 as d_year] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_customer_sk@5, ss_cdemo_sk@6, ss_hdemo_sk@7, ss_addr_sk@8, ss_store_sk@9, ss_promo_sk@10, ss_wholesale_cost@11, ss_list_price@12, ss_coupon_amt@13] │ CoalescePartitionsExec - │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 12] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, ss_item_sk@1)], projection=[ss_sold_date_sk@1, ss_item_sk@2, ss_customer_sk@3, ss_cdemo_sk@4, ss_hdemo_sk@5, ss_addr_sk@6, ss_store_sk@7, ss_promo_sk@8, ss_wholesale_cost@9, ss_list_price@10, ss_coupon_amt@11] │ CoalescePartitionsExec - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: CAST(sum(catalog_sales.cs_ext_list_price)@1 AS Decimal128(38, 2)) > Some(2),20,0 * sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)@2, projection=[cs_item_sk@0] - │ AggregateExec: mode=FinalPartitioned, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(catalog_sales.cs_ext_list_price), sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)] - │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 16] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_item_sk@0, ss_item_sk@1), (sr_ticket_number@1, ss_ticket_number@8)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_customer_sk@4, ss_cdemo_sk@5, ss_hdemo_sk@6, ss_addr_sk@7, ss_store_sk@8, ss_promo_sk@9, ss_wholesale_cost@11, ss_list_price@12, ss_coupon_amt@13] - │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 17] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 18] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_cdemo_sk@1 as c_current_cdemo_sk, c_current_hdemo_sk@2 as c_current_hdemo_sk, c_current_addr_sk@3 as c_current_addr_sk, c_first_shipto_date_sk@4 as c_first_shipto_date_sk, c_first_sales_date_sk@5 as c_first_sales_date_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet @@ -6770,51 +6764,57 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_item_sk@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(catalog_sales.cs_ext_list_price), sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)] - │ ProjectionExec: expr=[cs_item_sk@3 as cs_item_sk, cs_ext_list_price@4 as cs_ext_list_price, cr_refunded_cash@0 as cr_refunded_cash, cr_reversed_charge@1 as cr_reversed_charge, cr_store_credit@2 as cr_store_credit] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cr_item_sk@0, cs_item_sk@0), (cr_order_number@1, cs_order_number@1)], projection=[cr_refunded_cash@2, cr_reversed_charge@3, cr_store_credit@4, cs_item_sk@5, cs_ext_list_price@7] - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ FilterExec: CAST(sum(catalog_sales.cs_ext_list_price)@1 AS Decimal128(38, 2)) > Some(2),20,0 * sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)@2, projection=[cs_item_sk@0] + │ AggregateExec: mode=FinalPartitioned, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(catalog_sales.cs_ext_list_price), sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cr_item_sk@0, cr_order_number@1], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_refunded_cash, cr_reversed_charge, cr_store_credit], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_item_sk@0, cs_order_number@1], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ RepartitionExec: partitioning=Hash([cs_item_sk@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(catalog_sales.cs_ext_list_price), sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)] + │ ProjectionExec: expr=[cs_item_sk@3 as cs_item_sk, cs_ext_list_price@4 as cs_ext_list_price, cr_refunded_cash@0 as cr_refunded_cash, cr_reversed_charge@1 as cr_reversed_charge, cr_store_credit@2 as cr_store_credit] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cr_item_sk@0, cs_item_sk@0), (cr_order_number@1, cs_order_number@1)], projection=[cr_refunded_cash@2, cr_reversed_charge@3, cr_store_credit@4, cs_item_sk@5, cs_ext_list_price@7] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cr_item_sk@0, cr_order_number@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_refunded_cash, cr_reversed_charge, cr_store_credit], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@0, cs_order_number@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_item_sk@1, ss_ticket_number@8], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_item_sk@1, ss_ticket_number@8], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 9 ── Tasks: t0:[p0..p1] t1:[p2..p3] + ┌───── Stage 10 ── Tasks: t0:[p0..p1] t1:[p2..p3] │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 10 ── Tasks: t0:[p0..p1] t1:[p2..p3] + ┌───── Stage 11 ── Tasks: t0:[p0..p1] t1:[p2..p3] │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_name@1 as s_store_name, s_zip@2 as s_zip, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_zip], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: d_year@1 = 2000 @@ -6822,38 +6822,44 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] └────────────────────────────────────────────────── - ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_item_sk@0], 3), input_partitions=3 - │ AggregateExec: mode=Partial, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(catalog_sales.cs_ext_list_price), sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)] - │ ProjectionExec: expr=[cs_item_sk@3 as cs_item_sk, cs_ext_list_price@4 as cs_ext_list_price, cr_refunded_cash@0 as cr_refunded_cash, cr_reversed_charge@1 as cr_reversed_charge, cr_store_credit@2 as cr_store_credit] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cr_item_sk@0, cs_item_sk@0), (cr_order_number@1, cs_order_number@1)], projection=[cr_refunded_cash@2, cr_reversed_charge@3, cr_store_credit@4, cs_item_sk@5, cs_ext_list_price@7] - │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ FilterExec: CAST(sum(catalog_sales.cs_ext_list_price)@1 AS Decimal128(38, 2)) > Some(2),20,0 * sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)@2, projection=[cs_item_sk@0] + │ AggregateExec: mode=FinalPartitioned, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(catalog_sales.cs_ext_list_price), sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)] + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 12 ── Tasks: t0:[p0..p5] t1:[p0..p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cr_item_sk@0, cr_order_number@1], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_refunded_cash, cr_reversed_charge, cr_store_credit], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 13 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 15 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_item_sk@0, cs_order_number@1], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ RepartitionExec: partitioning=Hash([cs_item_sk@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(catalog_sales.cs_ext_list_price), sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)] + │ ProjectionExec: expr=[cs_item_sk@3 as cs_item_sk, cs_ext_list_price@4 as cs_ext_list_price, cr_refunded_cash@0 as cr_refunded_cash, cr_reversed_charge@1 as cr_reversed_charge, cr_store_credit@2 as cr_store_credit] + │ CoalesceBatchesExec: target_batch_size=8192 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cr_item_sk@0, cs_item_sk@0), (cr_order_number@1, cs_order_number@1)], projection=[cr_refunded_cash@2, cr_reversed_charge@3, cr_store_credit@4, cs_item_sk@5, cs_ext_list_price@7] + │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 15 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 13 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cr_item_sk@0, cr_order_number@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_refunded_cash, cr_reversed_charge, cr_store_credit], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([cs_item_sk@0, cs_order_number@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 17 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 18 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_item_sk@1, ss_ticket_number@8], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_item_sk@1, ss_ticket_number@8], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); @@ -7373,7 +7379,7 @@ mod tests { │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, cs_bill_cdemo_sk@2)], projection=[cs_sold_date_sk@2, cs_ship_date_sk@3, cs_bill_hdemo_sk@5, cs_item_sk@6, cs_promo_sk@7, cs_order_number@8, inv_date_sk@9, w_warehouse_name@10, i_item_desc@11] - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([cs_bill_cdemo_sk@2], 3), input_partitions=3 │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_ship_date_sk@2 as cs_ship_date_sk, cs_bill_cdemo_sk@3 as cs_bill_cdemo_sk, cs_bill_hdemo_sk@4 as cs_bill_hdemo_sk, cs_item_sk@5 as cs_item_sk, cs_promo_sk@6 as cs_promo_sk, cs_order_number@7 as cs_order_number, inv_date_sk@8 as inv_date_sk, w_warehouse_name@9 as w_warehouse_name, i_item_desc@0 as i_item_desc] @@ -7388,8 +7394,8 @@ mod tests { │ ProjectionExec: expr=[cs_sold_date_sk@2 as cs_sold_date_sk, cs_ship_date_sk@3 as cs_ship_date_sk, cs_bill_cdemo_sk@4 as cs_bill_cdemo_sk, cs_bill_hdemo_sk@5 as cs_bill_hdemo_sk, cs_item_sk@6 as cs_item_sk, cs_promo_sk@7 as cs_promo_sk, cs_order_number@8 as cs_order_number, inv_date_sk@0 as inv_date_sk, inv_warehouse_sk@1 as inv_warehouse_sk] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(inv_item_sk@1, cs_item_sk@4)], filter=inv_quantity_on_hand@1 < cs_quantity@0, projection=[inv_date_sk@0, inv_warehouse_sk@2, cs_sold_date_sk@4, cs_ship_date_sk@5, cs_bill_cdemo_sk@6, cs_bill_hdemo_sk@7, cs_item_sk@8, cs_promo_sk@9, cs_order_number@10] - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] @@ -7418,13 +7424,13 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_buy_potential], file_type=parquet, predicate=hd_buy_potential@1 = >10000, pruning_predicate=hd_buy_potential_null_count@2 != row_count@3 AND hd_buy_potential_min@0 <= >10000 AND >10000 <= hd_buy_potential_max@1, required_guarantees=[hd_buy_potential in (>10000)] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=3 + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 3), input_partitions=2 │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: cd_marital_status@1 = D, projection=[cd_demo_sk@0] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=cd_marital_status@1 = D, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= D AND D <= cd_marital_status_max@1, required_guarantees=[cd_marital_status in (D)] └────────────────────────────────────────────────── ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] @@ -7434,16 +7440,16 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_desc], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([inv_item_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([inv_item_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], ...]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_item_sk@4], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_item_sk@4], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_bill_cdemo_sk, cs_bill_hdemo_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); @@ -7541,7 +7547,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@4, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, ws_sold_date_sk@5, ws_net_paid@7] │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)] │ CoalescePartitionsExec @@ -7561,7 +7567,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@4, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, ss_sold_date_sk@5, ss_net_paid@7] │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[c_customer_id@0 as customer_id, c_first_name@1 as customer_first_name, c_last_name@2 as customer_last_name, sum(store_sales.ss_net_paid)@4 as year_total] │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, d_year@3 as d_year], aggr=[sum(store_sales.ss_net_paid)], ordering_mode=PartiallySorted([3]) │ CoalesceBatchesExec: target_batch_size=8192 @@ -7575,7 +7581,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@4, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, ss_sold_date_sk@5, ss_net_paid@7] │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_net_paid)@4 as year_total] │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, d_year@3 as d_year], aggr=[sum(web_sales.ws_net_paid)], ordering_mode=PartiallySorted([3]) │ CoalesceBatchesExec: target_batch_size=8192 @@ -7589,7 +7595,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@4, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, ws_sold_date_sk@5, ws_net_paid@7] │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] @@ -7607,10 +7613,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -7629,10 +7635,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -7651,10 +7657,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -7673,10 +7679,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); @@ -7970,7 +7976,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_category@1, ss_sold_date_sk@2, ss_ext_sales_price@4] │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 @@ -7979,12 +7985,12 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([ss_item_sk@1], 6), input_partitions=2 │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ss_store_sk@2 IS NULL, projection=[ss_sold_date_sk@0, ss_item_sk@1, ss_ext_sales_price@3] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=ss_store_sk@2 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=ss_store_sk_null_count@0 > 0, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] @@ -8002,7 +8008,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_category@1, ws_sold_date_sk@2, ws_ext_sales_price@4] │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 @@ -8011,12 +8017,12 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([ws_item_sk@1], 6), input_partitions=2 │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: ws_ship_customer_sk@2 IS NULL, projection=[ws_sold_date_sk@0, ws_item_sk@1, ws_ext_sales_price@3] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ship_customer_sk, ws_ext_sales_price], file_type=parquet, predicate=ws_ship_customer_sk@2 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=ws_ship_customer_sk_null_count@0 > 0, required_guarantees=[] └────────────────────────────────────────────────── ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p0..p5] @@ -8034,7 +8040,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_category@1, cs_sold_date_sk@2, cs_ext_sales_price@4] │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 @@ -8043,12 +8049,12 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 11 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 11 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([cs_item_sk@1], 6), input_partitions=2 │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: cs_ship_addr_sk@1 IS NULL, projection=[cs_sold_date_sk@0, cs_item_sk@2, cs_ext_sales_price@3] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_ship_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=cs_ship_addr_sk@1 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=cs_ship_addr_sk_null_count@0 > 0, required_guarantees=[] └────────────────────────────────────────────────── "); @@ -8270,8 +8276,8 @@ mod tests { │ ProjectionExec: expr=[ss_sold_date_sk@1 as ss_sold_date_sk, ss_item_sk@2 as ss_item_sk, ss_customer_sk@3 as ss_customer_sk, ss_quantity@4 as ss_quantity, ss_wholesale_cost@5 as ss_wholesale_cost, ss_sales_price@6 as ss_sales_price, sr_ticket_number@0 as sr_ticket_number] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(sr_ticket_number@1, ss_ticket_number@3), (sr_item_sk@0, ss_item_sk@1)], projection=[sr_ticket_number@1, ss_sold_date_sk@2, ss_item_sk@3, ss_customer_sk@4, ss_quantity@6, ss_wholesale_cost@7, ss_sales_price@8] - │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[d_year@0 as ws_sold_year, ws_item_sk@1 as ws_item_sk, ws_bill_customer_sk@2 as ws_customer_sk, sum(web_sales.ws_quantity)@3 as ws_qty, sum(web_sales.ws_wholesale_cost)@4 as ws_wc, sum(web_sales.ws_sales_price)@5 as ws_sp] │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, ws_item_sk@1 as ws_item_sk, ws_bill_customer_sk@2 as ws_bill_customer_sk], aggr=[sum(web_sales.ws_quantity), sum(web_sales.ws_wholesale_cost), sum(web_sales.ws_sales_price)] │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 @@ -8287,16 +8293,16 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@1 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_ticket_number@3, ss_item_sk@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_ticket_number@3, ss_item_sk@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_sales_price], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] @@ -8307,7 +8313,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@0)], projection=[d_year@1, ws_item_sk@4, ws_bill_customer_sk@5, ws_quantity@6, ws_wholesale_cost@7, ws_sales_price@8] │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 @@ -8317,7 +8323,7 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([ws_sold_date_sk@0], 6), input_partitions=3 │ CoalesceBatchesExec: target_batch_size=8192 @@ -8326,19 +8332,19 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(wr_order_number@1, ws_order_number@3), (wr_item_sk@0, ws_item_sk@1)], projection=[wr_order_number@1, ws_sold_date_sk@2, ws_item_sk@3, ws_bill_customer_sk@4, ws_quantity@6, ws_wholesale_cost@7, ws_sales_price@8] │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([wr_order_number@1, wr_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([wr_order_number@1, wr_item_sk@0], 9), input_partitions=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_order_number], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_order_number@3, ws_item_sk@1], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ws_order_number@3, ws_item_sk@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_order_number, ws_quantity, ws_wholesale_cost, ws_sales_price], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 13 ── Tasks: t0:[p0..p2] t1:[p0..p2] @@ -8349,7 +8355,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@0)], projection=[d_year@1, cs_bill_customer_sk@4, cs_item_sk@5, cs_quantity@6, cs_wholesale_cost@7, cs_sales_price@8] │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 @@ -8359,7 +8365,7 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 12 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 12 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([cs_sold_date_sk@0], 6), input_partitions=3 │ CoalesceBatchesExec: target_batch_size=8192 @@ -8367,19 +8373,19 @@ mod tests { │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_bill_customer_sk@2 as cs_bill_customer_sk, cs_item_sk@3 as cs_item_sk, cs_quantity@4 as cs_quantity, cs_wholesale_cost@5 as cs_wholesale_cost, cs_sales_price@6 as cs_sales_price, cr_order_number@0 as cr_order_number] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(cr_order_number@1, cs_order_number@3), (cr_item_sk@0, cs_item_sk@2)], projection=[cr_order_number@1, cs_sold_date_sk@2, cs_bill_customer_sk@3, cs_item_sk@4, cs_quantity@6, cs_wholesale_cost@7, cs_sales_price@8] - │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 10 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cr_order_number@1, cr_item_sk@0], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cr_order_number@1, cr_item_sk@0], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 11 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 11 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_order_number@3, cs_item_sk@2], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_order_number@3, cs_item_sk@2], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_order_number, cs_quantity, cs_wholesale_cost, cs_sales_price], file_type=parquet └────────────────────────────────────────────────── "); @@ -8488,8 +8494,8 @@ mod tests { │ ProjectionExec: expr=[ss_sold_date_sk@2 as ss_sold_date_sk, ss_item_sk@3 as ss_item_sk, ss_store_sk@4 as ss_store_sk, ss_promo_sk@5 as ss_promo_sk, ss_ext_sales_price@6 as ss_ext_sales_price, ss_net_profit@7 as ss_net_profit, sr_return_amt@0 as sr_return_amt, sr_net_loss@1 as sr_net_loss] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(sr_item_sk@0, ss_item_sk@1), (sr_ticket_number@1, ss_ticket_number@4)], projection=[sr_return_amt@2, sr_net_loss@3, ss_sold_date_sk@4, ss_item_sk@5, ss_store_sk@6, ss_promo_sk@7, ss_ext_sales_price@9, ss_net_profit@10] - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[catalog channel as channel, concat(catalog_page, catalog_page_id@0) as id, sales@1 as sales, returns_@2 as returns_, profit@3 as profit] │ ProjectionExec: expr=[cp_catalog_page_id@0 as catalog_page_id, sum(catalog_sales.cs_ext_sales_price)@1 as sales, sum(coalesce(catalog_returns.cr_return_amount,Int64(0)))@2 as returns_, sum(catalog_sales.cs_net_profit - coalesce(catalog_returns.cr_net_loss,Int64(0)))@3 as profit] │ AggregateExec: mode=FinalPartitioned, gby=[cp_catalog_page_id@0 as cp_catalog_page_id], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(coalesce(catalog_returns.cr_return_amount,Int64(0))), sum(catalog_sales.cs_net_profit - coalesce(catalog_returns.cr_net_loss,Int64(0)))] @@ -8515,8 +8521,8 @@ mod tests { │ ProjectionExec: expr=[cs_sold_date_sk@2 as cs_sold_date_sk, cs_catalog_page_sk@3 as cs_catalog_page_sk, cs_item_sk@4 as cs_item_sk, cs_promo_sk@5 as cs_promo_sk, cs_ext_sales_price@6 as cs_ext_sales_price, cs_net_profit@7 as cs_net_profit, cr_return_amount@0 as cr_return_amount, cr_net_loss@1 as cr_net_loss] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(cr_item_sk@0, cs_item_sk@2), (cr_order_number@1, cs_order_number@4)], projection=[cr_return_amount@2, cr_net_loss@3, cs_sold_date_sk@4, cs_catalog_page_sk@5, cs_item_sk@6, cs_promo_sk@7, cs_ext_sales_price@9, cs_net_profit@10] - │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ ProjectionExec: expr=[web channel as channel, concat(web_site, web_site_id@0) as id, sales@1 as sales, returns_@2 as returns_, profit@3 as profit] │ ProjectionExec: expr=[web_site_id@0 as web_site_id, sum(web_sales.ws_ext_sales_price)@1 as sales, sum(coalesce(web_returns.wr_return_amt,Int64(0)))@2 as returns_, sum(web_sales.ws_net_profit - coalesce(web_returns.wr_net_loss,Int64(0)))@3 as profit] @@ -8543,7 +8549,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(wr_item_sk@0, ws_item_sk@1), (wr_order_number@1, ws_order_number@4)], projection=[wr_return_amt@2, wr_net_loss@3, ws_sold_date_sk@4, ws_item_sk@5, ws_web_site_sk@6, ws_promo_sk@7, ws_ext_sales_price@9, ws_net_profit@10] │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: i_current_price@1 > Some(5000),4,2, projection=[i_item_sk@0] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 @@ -8582,16 +8588,16 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number, sr_return_amt, sr_net_loss], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_item_sk@1, ss_ticket_number@4], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_item_sk@1, ss_ticket_number@4], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_ext_sales_price, ss_net_profit], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -8617,16 +8623,16 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cr_item_sk@0, cr_order_number@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cr_item_sk@0, cr_order_number@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_return_amount, cr_net_loss], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cs_item_sk@2, cs_order_number@4], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cs_item_sk@2, cs_order_number@4], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_catalog_page_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_ext_sales_price, cs_net_profit], file_type=parquet └────────────────────────────────────────────────── ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] @@ -8657,10 +8663,10 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_item_sk@1, ws_order_number@4], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ws_item_sk@1, ws_order_number@4], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_site_sk, ws_promo_sk, ws_order_number, ws_ext_sales_price, ws_net_profit], file_type=parquet └────────────────────────────────────────────────── "); @@ -8771,7 +8777,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(inv_item_sk@1, i_item_sk@0)], projection=[inv_date_sk@0, i_item_sk@2, i_item_id@3, i_item_desc@4, i_current_price@5] │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: i_current_price@3 >= Some(6200),4,2 AND i_current_price@3 <= Some(9200),4,2 AND i_manufact_id@4 IN (SET) ([129, 270, 821, 423]), projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3] │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 @@ -8782,10 +8788,10 @@ mod tests { │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 2000-05-25 AND d_date@1 <= 2000-07-24 AND DynamicFilter [ empty ], pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-05-25 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-07-24, required_guarantees=[] │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: inv_quantity_on_hand@2 >= 100 AND inv_quantity_on_hand@2 <= 500, projection=[inv_date_sk@0, inv_item_sk@1] - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], ...]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@2 >= 100 AND inv_quantity_on_hand@2 <= 500, pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] └────────────────────────────────────────────────── "); @@ -8845,7 +8851,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, sr_returned_date_sk@0)], projection=[d_date@1, sr_return_quantity@4, i_item_id@5] │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[i_item_id@0 as item_id, sum(catalog_returns.cr_return_quantity)@1 as cr_item_qty] │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(catalog_returns.cr_return_quantity)] │ CoalesceBatchesExec: target_batch_size=8192 @@ -8864,7 +8870,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(d_date_sk@0, cr_returned_date_sk@0)], projection=[d_date@1, cr_return_quantity@3, i_item_id@4] │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ ProjectionExec: expr=[wr_returned_date_sk@1 as wr_returned_date_sk, wr_return_quantity@2 as wr_return_quantity, i_item_id@0 as i_item_id] @@ -8909,26 +8915,26 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([sr_returned_date_sk@0], 3), input_partitions=3 │ ProjectionExec: expr=[sr_returned_date_sk@1 as sr_returned_date_sk, sr_return_quantity@2 as sr_return_quantity, i_item_id@0 as i_item_id] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, sr_item_sk@1)], projection=[i_item_id@1, sr_returned_date_sk@2, sr_return_quantity@4] │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 9), input_partitions=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 8 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([sr_item_sk@1], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([sr_item_sk@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_returned_date_sk, sr_item_sk, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] @@ -8945,26 +8951,26 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 │ RepartitionExec: partitioning=Hash([cr_returned_date_sk@0], 3), input_partitions=3 │ ProjectionExec: expr=[cr_returned_date_sk@1 as cr_returned_date_sk, cr_return_quantity@2 as cr_return_quantity, i_item_id@0 as i_item_id] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@0, cr_item_sk@1)], projection=[i_item_id@1, cr_returned_date_sk@2, cr_return_quantity@4] │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 12 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 12 ── Tasks: t0:[p0..p8] t1:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([i_item_sk@0], 6), input_partitions=3 + │ RepartitionExec: partitioning=Hash([i_item_sk@0], 9), input_partitions=3 │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 13 ── Tasks: t0:[p0..p5] t1:[p0..p5] + ┌───── Stage 13 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([cr_item_sk@1], 6), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([cr_item_sk@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_returned_date_sk, cr_item_sk, cr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); @@ -9065,7 +9071,7 @@ mod tests { │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(wr_item_sk@0, ws_item_sk@1), (wr_order_number@5, ws_order_number@3)], projection=[wr_refunded_cdemo_sk@1, wr_refunded_addr_sk@2, wr_returning_cdemo_sk@3, wr_reason_sk@4, wr_fee@6, wr_refunded_cash@7, ws_sold_date_sk@8, ws_web_page_sk@10, ws_quantity@12, ws_sales_price@13, ws_net_profit@14] │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, CAST(cd_demo_sk@0 AS Float64) as CAST(cd1.cd_demo_sk AS Float64)] │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: cd_marital_status@1 = M AND cd_education_status@2 = Advanced Degree OR cd_marital_status@1 = S AND cd_education_status@2 = College OR cd_marital_status@1 = W AND cd_education_status@2 = 2 yr Degree @@ -9100,12 +9106,12 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_refunded_cdemo_sk, wr_refunded_addr_sk, wr_returning_cdemo_sk, wr_reason_sk, wr_order_number, wr_fee, wr_refunded_cash], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_item_sk@1, ws_order_number@3], 3), input_partitions=3 + │ RepartitionExec: partitioning=Hash([ws_item_sk@1, ws_order_number@3], 3), input_partitions=2 │ CoalesceBatchesExec: target_batch_size=8192 │ FilterExec: (ws_net_profit@6 >= Some(10000),7,2 AND ws_net_profit@6 <= Some(20000),7,2 OR ws_net_profit@6 >= Some(15000),7,2 AND ws_net_profit@6 <= Some(30000),7,2 OR ws_net_profit@6 >= Some(5000),7,2 AND ws_net_profit@6 <= Some(25000),7,2) AND (ws_sales_price@5 >= Some(10000),5,2 AND ws_sales_price@5 <= Some(15000),5,2 OR ws_sales_price@5 >= Some(5000),5,2 AND ws_sales_price@5 <= Some(10000),5,2 OR ws_sales_price@5 >= Some(15000),5,2 AND ws_sales_price@5 <= Some(20000),5,2) - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_page_sk, ws_order_number, ws_quantity, ws_sales_price, ws_net_profit], file_type=parquet, predicate=(ws_net_profit@6 >= Some(10000),7,2 AND ws_net_profit@6 <= Some(20000),7,2 OR ws_net_profit@6 >= Some(15000),7,2 AND ws_net_profit@6 <= Some(30000),7,2 OR ws_net_profit@6 >= Some(5000),7,2 AND ws_net_profit@6 <= Some(25000),7,2) AND (ws_sales_price@5 >= Some(10000),5,2 AND ws_sales_price@5 <= Some(15000),5,2 OR ws_sales_price@5 >= Some(5000),5,2 AND ws_sales_price@5 <= Some(10000),5,2 OR ws_sales_price@5 >= Some(15000),5,2 AND ws_sales_price@5 <= Some(20000),5,2) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=(ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(10000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(20000),7,2 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(15000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(30000),7,2 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(5000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(25000),7,2) AND (ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(10000),5,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(15000),5,2 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(5000),5,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(10000),5,2 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(15000),5,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(20000),5,2), required_guarantees=[] └────────────────────────────────────────────────── "); @@ -9867,8 +9873,8 @@ mod tests { │ ProjectionExec: expr=[ss_customer_sk@2 as ss_customer_sk, ss_quantity@3 as ss_quantity, ss_sales_price@4 as ss_sales_price, sr_reason_sk@0 as sr_reason_sk, sr_return_quantity@1 as sr_return_quantity] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(sr_item_sk@0, ss_item_sk@0), (sr_ticket_number@2, ss_ticket_number@2)], projection=[sr_reason_sk@1, sr_return_quantity@3, ss_customer_sk@5, ss_quantity@7, ss_sales_price@8] - │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[r_reason_sk@0 as r_reason_sk, CAST(r_reason_sk@0 AS Float64) as CAST(reason.r_reason_sk AS Float64)] @@ -9878,16 +9884,16 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/reason/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-3.parquet]]}, projection=[r_reason_sk, r_reason_desc], file_type=parquet, predicate=r_reason_desc@1 = reason 28, pruning_predicate=r_reason_desc_null_count@2 != row_count@3 AND r_reason_desc_min@0 <= reason 28 AND reason 28 <= r_reason_desc_max@1, required_guarantees=[r_reason_desc in (reason 28)] └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@2], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_reason_sk, sr_ticket_number, sr_return_quantity], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ss_item_sk@0, ss_ticket_number@2], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ss_item_sk@0, ss_ticket_number@2], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_item_sk, ss_customer_sk, ss_ticket_number, ss_quantity, ss_sales_price], file_type=parquet └────────────────────────────────────────────────── "); @@ -9992,15 +9998,15 @@ mod tests { │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ws_order_number@1, ws_order_number@1)], filter=ws_warehouse_sk@1 != ws_warehouse_sk@0, projection=[ws_order_number@1] - │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(wr_order_number@0, ws_order_number@0)], projection=[wr_order_number@0] │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 │ CoalesceBatchesExec: target_batch_size=8192 │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ws_order_number@1, ws_order_number@1)], filter=ws_warehouse_sk@1 != ws_warehouse_sk@0, projection=[ws_order_number@1] - │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 - │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] │ ProjectionExec: expr=[web_site_sk@0 as web_site_sk, CAST(web_site_sk@0 AS Float64) as CAST(web_site.web_site_sk AS Float64)] @@ -10026,16 +10032,16 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@1 >= 1999-02-01 AND d_date@1 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_order_number@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ws_order_number@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_order_number@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ws_order_number@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] @@ -10045,16 +10051,16 @@ mod tests { │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_order_number], file_type=parquet └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_order_number@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ws_order_number@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ws_order_number@1], 3), input_partitions=3 - │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ RepartitionExec: partitioning=Hash([ws_order_number@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] └────────────────────────────────────────────────── "); From 1fb4daa5f89054f788b606b333720ba174906450 Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Fri, 2 Jan 2026 11:44:03 +0100 Subject: [PATCH 12/27] Adapt remote benchmarks to support more datasets (#276) * Vendor openssl in benchmarks * Support any dataset in benchmarks * Add TPC-DS schemas * Pass runtime env in Worker --- Cargo.lock | 11 + benchmarks/Cargo.toml | 1 + benchmarks/cdk/README.md | 2 +- benchmarks/cdk/bin/@bench-common.ts | 263 +++--- benchmarks/cdk/bin/datafusion-bench.ts | 34 +- benchmarks/cdk/bin/trino-bench.ts | 1014 +++++++++++++++++++----- benchmarks/cdk/bin/worker.rs | 17 +- src/flight_service/worker.rs | 7 + 8 files changed, 1002 insertions(+), 347 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7db559ee..c6e3043f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1918,6 +1918,7 @@ dependencies = [ "futures", "log", "object_store", + "openssl", "parquet", "prost", "serde", @@ -3763,6 +3764,15 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "openssl-src" +version = "300.5.4+3.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507b3792995dae9b0df8a1c1e3771e8418b7c2d9f0baeba32e6fe8b06c7cb72" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" version = "0.9.111" @@ -3771,6 +3781,7 @@ checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" dependencies = [ "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 204943e3..3c7394a9 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -27,6 +27,7 @@ axum = "0.7" object_store = { version = "0.12.4", features = ["aws"] } aws-config = "1" aws-sdk-ec2 = "1" +openssl = { version = "0.10", features = ["vendored"] } [[bin]] name = "dfbench" diff --git a/benchmarks/cdk/README.md b/benchmarks/cdk/README.md index 3d7cabb6..37b97a38 100644 --- a/benchmarks/cdk/README.md +++ b/benchmarks/cdk/README.md @@ -119,5 +119,5 @@ Several arguments can be passed for running the benchmarks against different sca for example: ```shell -npm run datafusion-bench -- --sf 10 --files-per-task 4 --query 7 +npm run datafusion-bench -- --datset tpch_sf10 --files-per-task 4 --query 7 ``` \ No newline at end of file diff --git a/benchmarks/cdk/bin/@bench-common.ts b/benchmarks/cdk/bin/@bench-common.ts index 2d978b60..a37fea62 100644 --- a/benchmarks/cdk/bin/@bench-common.ts +++ b/benchmarks/cdk/bin/@bench-common.ts @@ -1,135 +1,204 @@ import path from "path"; import fs from "fs/promises"; -import { z } from 'zod'; +import {z} from 'zod'; export const ROOT = path.join(__dirname, '../../..') +export const BUCKET = 's3://datafusion-distributed-benchmarks' // hardcoded in CDK code // Simple data structures export type QueryResult = { - query: string; - iterations: { elapsed: number; row_count: number }[]; + query: string; + iterations: { elapsed: number; row_count: number }[]; + failure?: string } export type BenchmarkResults = { - queries: QueryResult[]; + queries: QueryResult[]; } export const BenchmarkResults = z.object({ - queries: z.array(z.object({ - query: z.string(), - iterations: z.array(z.object({ - elapsed: z.number(), - row_count: z.number() + queries: z.array(z.object({ + query: z.string(), + iterations: z.array(z.object({ + elapsed: z.number(), + row_count: z.number() + })), + failed: z.string().optional() })) - })) }) -export const IDS = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22] - export async function writeJson(results: BenchmarkResults, outputPath?: string) { - if (!outputPath) return; - await fs.mkdir(path.dirname(outputPath), { recursive: true }); - await fs.writeFile(outputPath, JSON.stringify(results, null, 2)); + if (!outputPath) return; + await fs.mkdir(path.dirname(outputPath), { recursive: true }); + await fs.writeFile(outputPath, JSON.stringify(results, null, 2)); } export async function compareWithPrevious(results: BenchmarkResults, outputPath: string) { - let prevResults: BenchmarkResults; - try { - const prevContent = await fs.readFile(outputPath, 'utf-8'); - prevResults = BenchmarkResults.parse(JSON.parse(prevContent)); - } catch { - return; // No previous results to compare - } - - console.log('\n==== Comparison with previous run ===='); - - for (const query of results.queries) { - const prevQuery = prevResults.queries.find(q => q.query === query.query); - if (!prevQuery || prevQuery.iterations.length === 0 || query.iterations.length === 0) { - continue; + let prevResults: BenchmarkResults; + try { + const prevContent = await fs.readFile(outputPath, 'utf-8'); + prevResults = BenchmarkResults.parse(JSON.parse(prevContent)); + } catch { + return; // No previous results to compare } - const avgPrev = Math.round( - prevQuery.iterations.reduce((sum, i) => sum + i.elapsed, 0) / prevQuery.iterations.length - ); - const avg = Math.round( - query.iterations.reduce((sum, i) => sum + i.elapsed, 0) / query.iterations.length - ); - - const factor = avg < avgPrev ? avgPrev / avg : avg / avgPrev; - const tag = avg < avgPrev ? "faster" : "slower"; - const emoji = factor > 1.2 ? (avg < avgPrev ? "✅" : "❌") : (avg < avgPrev ? "✔" : "✖"); - - console.log( - `${query.query.padStart(8)}: prev=${avgPrev.toString().padStart(4)} ms, new=${avg.toString().padStart(4)} ms, ${factor.toFixed(2)}x ${tag} ${emoji}` - ); - } -} - -export interface BenchmarkRunner { - createTables(sf: number): Promise; - - executeQuery(query: string): Promise<{ rowCount: number }>; -} + console.log('\n==== Comparison with previous run ===='); -export async function runBenchmark( - runner: BenchmarkRunner, - options: { - sf: number; - iterations: number; - specificQuery?: number; - outputPath: string; - } -) { - const { sf, iterations, specificQuery, outputPath } = options; + for (const query of results.queries) { + const prevQuery = prevResults.queries.find(q => q.query === query.query); + if (!prevQuery || prevQuery.iterations.length === 0 || query.iterations.length === 0) { + continue; + } - const results: BenchmarkResults = { queries: [] }; - const queriesPath = path.join(ROOT, "testdata", "tpch", "queries") + const avgPrev = Math.round( + prevQuery.iterations.reduce((sum, i) => sum + i.elapsed, 0) / prevQuery.iterations.length + ); + const avg = Math.round( + query.iterations.reduce((sum, i) => sum + i.elapsed, 0) / query.iterations.length + ); - console.log("Creating tables..."); - await runner.createTables(sf); + const factor = avg < avgPrev ? avgPrev / avg : avg / avgPrev; + const tag = avg < avgPrev ? "faster" : "slower"; + const emoji = factor > 1.2 ? (avg < avgPrev ? "✅" : "❌") : (avg < avgPrev ? "✔" : "✖"); - for (let id of IDS) { - if (specificQuery && specificQuery !== id) { - continue; + console.log( + `${query.query.padStart(8)}: prev=${avgPrev.toString().padStart(4)} ms, new=${avg.toString().padStart(4)} ms, ${factor.toFixed(2)}x ${tag} ${emoji}` + ); } +} - const queryId = `q${id}`; - const filePath = path.join(queriesPath, `${queryId}.sql`) - const queryToExecute = await fs.readFile(filePath, 'utf-8') +export interface TableSpec { + schema: string + name: string + s3Path: string +} + +export interface BenchmarkRunner { + createTables(s3Paths: TableSpec[]): Promise; - const queryResult: QueryResult = { - query: queryId, - iterations: [] - }; + executeQuery(query: string): Promise<{ rowCount: number }>; +} - console.log(`Warming up query ${id}...`) - await runner.executeQuery(queryToExecute); +async function tablePathsForDataset(dataset: string): Promise { + const datasetPath = path.join(ROOT, "benchmarks", "data", dataset) + + const result: TableSpec[] = [] + for (const entryName of await fs.readdir(datasetPath)) { + const dir = path.join(datasetPath, entryName) + if (await isDirWithAllParquetFiles(dir)) { + result.push({ + name: entryName, + schema: dataset, + s3Path: `${BUCKET}/${dataset}/${entryName}/` + }) + } + } + return result +} - for (let i = 0; i < iterations; i++) { - const start = new Date() - const response = await runner.executeQuery(queryToExecute); - const elapsed = Math.round(new Date().getTime() - start.getTime()) +async function isDirWithAllParquetFiles(dir: string): Promise { + let readDir + try { + readDir = await fs.readdir(dir) + } catch (e) { + return false + } + for (const file of readDir) { + if (!file.endsWith(".parquet")) { + return false + } + } + return true +} - queryResult.iterations.push({ - elapsed, - row_count: response.rowCount - }); +async function queriesForDataset(dataset: string): Promise<[string, string][]> { + const datasetSuffix = dataset.split("_")[0] + const queriesPath = path.join(ROOT, "testdata", datasetSuffix, "queries") - console.log( - `Query ${id} iteration ${i} took ${elapsed} ms and returned ${response.rowCount} rows` - ); + const queries: [string, string][] = [] + for (const queryName of await fs.readdir(queriesPath)) { + const sql = await fs.readFile(path.join(queriesPath, queryName), 'utf-8'); + queries.push([queryName, sql]) } + queries.sort(([name1], [name2]) => numericId(name1) > numericId(name2) ? 1 : -1) + return queries +} - const avg = Math.round( - queryResult.iterations.reduce((a, b) => a + b.elapsed, 0) / queryResult.iterations.length - ); - console.log(`Query ${id} avg time: ${avg} ms`); +function numericId(queryName: string): number { + return parseInt([...queryName.matchAll(/(\d+)/g)][0][0]) +} - results.queries.push(queryResult); - } +export async function runBenchmark( + runner: BenchmarkRunner, + options: { + dataset: string + iterations: number; + queries: number[]; + outputPath: string; + } +) { + const { dataset, iterations, queries, outputPath } = options; + + const results: BenchmarkResults = { queries: [] }; + + console.log("Creating tables..."); + const s3Paths = await tablePathsForDataset(dataset) + await runner.createTables(s3Paths); + + for (const [queryName, sql] of await queriesForDataset(dataset)) { + const id = numericId(queryName) + + if (queries.length > 0 && !queries.includes(id)) { + continue; + } + + const queryResult: QueryResult = { + query: queryName, + iterations: [], + }; + + console.log(`Warming up query ${id}...`) + try { + await runner.executeQuery(sql); + } catch (e: any) { + queryResult.failure = e.toString(); + console.error(`Query ${queryResult.query} failed: ${queryResult.failure}`) + continue + } + + for (let i = 0; i < iterations; i++) { + const start = new Date() + let response + try { + response = await runner.executeQuery(sql); + } catch (e: any) { + queryResult.failure = e.toString(); + break + } + const elapsed = Math.round(new Date().getTime() - start.getTime()) + + queryResult.iterations.push({ + elapsed, + row_count: response.rowCount + }); + + console.log( + `Query ${id} iteration ${i} took ${elapsed} ms and returned ${response.rowCount} rows` + ); + } + + const avg = Math.round( + queryResult.iterations.reduce((a, b) => a + b.elapsed, 0) / queryResult.iterations.length + ); + console.log(`Query ${id} avg time: ${avg} ms`); + + if (queryResult.failure) { + console.error(`Query ${queryResult.query} failed: ${queryResult.failure}`) + } + results.queries.push(queryResult); + } - // Write results and compare - await compareWithPrevious(results, outputPath); - await writeJson(results, outputPath); + // Write results and compare + await compareWithPrevious(results, outputPath); + await writeJson(results, outputPath); } diff --git a/benchmarks/cdk/bin/datafusion-bench.ts b/benchmarks/cdk/bin/datafusion-bench.ts index 4c5b3409..c817a71c 100644 --- a/benchmarks/cdk/bin/datafusion-bench.ts +++ b/benchmarks/cdk/bin/datafusion-bench.ts @@ -1,7 +1,7 @@ import path from "path"; import {Command} from "commander"; import {z} from 'zod'; -import {BenchmarkRunner, ROOT, runBenchmark} from "./@bench-common"; +import {BenchmarkRunner, ROOT, runBenchmark, TableSpec} from "./@bench-common"; // Remember to port-forward a worker with // aws ssm start-session --target {host-id} --document-name AWS-StartPortForwardingSession --parameters "portNumber=9000,localPortNumber=9000" @@ -10,7 +10,7 @@ async function main() { const program = new Command(); program - .option('--sf ', 'Scale factor', '1') + .option('--dataset ', 'Dataset to run queries on') .option('-i, --iterations ', 'Number of iterations', '3') .option('--files-per-task ', 'Files per task', '4') .option('--cardinality-task-sf ', 'Cardinality task scale factor', '2') @@ -21,12 +21,12 @@ async function main() { const options = program.opts(); - const sf = parseInt(options.sf); + const dataset: string = options.dataset const iterations = parseInt(options.iterations); const filesPerTask = parseInt(options.filesPerTask); const cardinalityTaskSf = parseInt(options.cardinalityTaskSf); const shuffleBatchSize = parseInt(options.shuffleBatchSize); - const specificQuery = options.query ? parseInt(options.query) : undefined; + const queries = options.query ? [parseInt(options.query)] : []; const collectMetrics = options.collectMetrics === 'true' || options.collectMetrics === 1 const runner = new DataFusionRunner({ @@ -36,12 +36,13 @@ async function main() { collectMetrics }); - const outputPath = path.join(ROOT, "benchmarks", "data", `tpch_sf${sf}`, "remote-results.json"); + const datasetPath = path.join(ROOT, "benchmarks", "data", dataset); + const outputPath = path.join(datasetPath, "remote-results.json") await runBenchmark(runner, { - sf, + dataset, iterations, - specificQuery, + queries, outputPath, }); } @@ -75,7 +76,7 @@ class DataFusionRunner implements BenchmarkRunner { response = await this.query(sql) } - return {rowCount: response.count}; + return { rowCount: response.count }; } private async query(sql: string): Promise { @@ -93,22 +94,13 @@ class DataFusionRunner implements BenchmarkRunner { return QueryResponse.parse(unparsed); } - async createTables(sf: number): Promise { + async createTables(tables: TableSpec[]): Promise { let stmt = ''; - for (const tbl of [ - "lineitem", - "orders", - "part", - "partsupp", - "customer", - "nation", - "region", - "supplier", - ]) { + for (const table of tables) { // language=SQL format=false stmt += ` - DROP TABLE IF EXISTS ${tbl}; - CREATE EXTERNAL TABLE IF NOT EXISTS ${tbl} STORED AS PARQUET LOCATION 's3://datafusion-distributed-benchmarks/tpch_sf${sf}/${tbl}/'; + DROP TABLE IF EXISTS ${table.name}; + CREATE EXTERNAL TABLE IF NOT EXISTS ${table.name} STORED AS PARQUET LOCATION '${table.s3Path}'; `; } await this.query(stmt); diff --git a/benchmarks/cdk/bin/trino-bench.ts b/benchmarks/cdk/bin/trino-bench.ts index d3536374..136b3a0f 100644 --- a/benchmarks/cdk/bin/trino-bench.ts +++ b/benchmarks/cdk/bin/trino-bench.ts @@ -1,249 +1,825 @@ import path from "path"; -import { Command } from "commander"; -import { ROOT, runBenchmark, BenchmarkRunner } from "./@bench-common"; +import {Command} from "commander"; +import {ROOT, runBenchmark, BenchmarkRunner, TableSpec} from "./@bench-common"; // Remember to port-forward Trino coordinator with // aws ssm start-session --target {instance-0-id} --document-name AWS-StartPortForwardingSession --parameters "portNumber=8080,localPortNumber=8080" async function main() { - const program = new Command(); + const program = new Command(); - program - .option('--sf ', 'Scale factor', '1') - .option('-i, --iterations ', 'Number of iterations', '3') - .option('--query ', 'A specific query to run', undefined) - .parse(process.argv); + program + .option('--dataset ', 'Scale factor', '1') + .option('-i, --iterations ', 'Number of iterations', '3') + .option('--query ', 'A specific query to run', undefined) + .parse(process.argv); - const options = program.opts(); + const options = program.opts(); - const sf = parseInt(options.sf); - const iterations = parseInt(options.iterations); - const specificQuery = options.query ? parseInt(options.query) : undefined; + const dataset: string = options.dataset + const iterations = parseInt(options.iterations); + const queries = options.query ? [parseInt(options.query)] : []; - const runner = new TrinoRunner({ sf }); - const outputPath = path.join(ROOT, "benchmarks", "data", `tpch_sf${sf}`, "remote-results.json"); + const datasetPath = path.join(ROOT, "benchmarks", "data", dataset); + const outputPath = path.join(datasetPath, "remote-results.json") - await runBenchmark(runner, { - sf, - iterations, - specificQuery, - outputPath, - }); + const runner = new TrinoRunner(); + + await runBenchmark(runner, { + dataset, + iterations, + queries, + outputPath, + }); } class TrinoRunner implements BenchmarkRunner { - private trinoUrl = 'http://localhost:8080'; - - constructor(private readonly options: { - sf: number - }) { - } - - - async executeQuery(sql: string): Promise<{ rowCount: number }> { - // Fix query 4: Add DATE prefix to date literals that don't have it. - sql = sql.replace(/(? { + // Fix TPCH query 4: Add DATE prefix to date literals that don't have it. + sql = sql.replace(/(? { - // Submit query - const submitResponse = await fetch(`${this.trinoUrl}/v1/statement`, { - method: 'POST', - headers: { - 'X-Trino-User': 'benchmark', - 'X-Trino-Catalog': 'hive', - 'X-Trino-Schema': `tpch_sf${this.options.sf}`, - }, - body: sql.trim().replace(/;+$/, ''), - }); + private async executeSingleStatement(sql: string): Promise<{ rowCount: number }> { + if (!this.schema) { + throw new Error("No schema available, where the tables created?") + } + + // Submit query + const submitResponse = await fetch(`${this.trinoUrl}/v1/statement`, { + method: 'POST', + headers: { + 'X-Trino-User': 'benchmark', + 'X-Trino-Catalog': 'hive', + 'X-Trino-Schema': this.schema ?? '', + }, + body: sql.trim().replace(/;+$/, ''), + }); + + if (!submitResponse.ok) { + const msg = await submitResponse.text(); + throw new Error(`Query submission failed: ${submitResponse.status} ${msg}`); + } - if (!submitResponse.ok) { - const msg = await submitResponse.text(); - throw new Error(`Query submission failed: ${submitResponse.status} ${msg}`); + let result: any = await submitResponse.json(); + let rowCount = 0; + + // Poll for results + while (result.nextUri) { + const pollResponse = await fetch(result.nextUri); + + if (!pollResponse.ok) { + const msg = await pollResponse.text(); + throw new Error(`Query polling failed: ${pollResponse.status} ${msg}`); + } + + result = await pollResponse.json(); + + // Count rows if data is present + if (result.data) { + if (typeof result.data?.[0]?.[0] === 'string') { + // Extract row count from EXPLAIN ANALYZE output + const outputMatch = result.data[0][0].match(/Output.*?(\d+)\s+rows/i); + if (outputMatch) { + rowCount = parseInt(outputMatch[1]); + } + } else { + rowCount += result.data.length; + } + } + + // Check for errors + if (result.error) { + throw new Error(`Query failed: ${result.error.message}`); + } + } + + return { rowCount }; } - let result: any = await submitResponse.json(); - let rowCount = 0; + async createTables(tables: TableSpec[]): Promise { + if (tables.length === 0) { + throw new Error("No table passed") + } + let schema = tables[0].schema + let basePath = tables[0].s3Path.split('/').slice(0, -1).join("/") - // Poll for results - while (result.nextUri) { - const pollResponse = await fetch(result.nextUri); + this.schema = schema - if (!pollResponse.ok) { - const msg = await pollResponse.text(); - throw new Error(`Query polling failed: ${pollResponse.status} ${msg}`); - } + await this.executeSingleStatement(` + CREATE SCHEMA IF NOT EXISTS hive."${schema}" WITH (location = '${basePath}')`); - result = await pollResponse.json(); + for (const table of tables) { + await this.executeSingleStatement(` + DROP TABLE IF EXISTS hive."${table.schema}"."${table.name}"`); - // Count rows if data is present - if (result.data) { - if (typeof result.data?.[0]?.[0] === 'string') { - // Extract row count from EXPLAIN ANALYZE output - const outputMatch = result.data[0][0].match(/Output.*?(\d+)\s+rows/i); - if (outputMatch) { - rowCount = parseInt(outputMatch[1]); - } - } else { - rowCount += result.data.length; + await this.executeSingleStatement(` + CREATE TABLE hive."${table.schema}"."${table.name}" ${getSchema(table)} + WITH (external_location = '${table.s3Path}', format = 'PARQUET')`); } - } + } +} - // Check for errors - if (result.error) { - throw new Error(`Query failed: ${result.error.message}`); - } +const SCHEMAS: Record> = { + tpch: { + customer: `( + c_custkey bigint, + c_name varchar(25), + c_address varchar(40), + c_nationkey bigint, + c_phone varchar(15), + c_acctbal decimal(15, 2), + c_mktsegment varchar(10), + c_comment varchar(117) +)`, + lineitem: `( + l_orderkey bigint, + l_partkey bigint, + l_suppkey bigint, + l_linenumber integer, + l_quantity decimal(15, 2), + l_extendedprice decimal(15, 2), + l_discount decimal(15, 2), + l_tax decimal(15, 2), + l_returnflag varchar(1), + l_linestatus varchar(1), + l_shipdate date, + l_commitdate date, + l_receiptdate date, + l_shipinstruct varchar(25), + l_shipmode varchar(10), + l_comment varchar(44) +)`, + nation: `( + n_nationkey bigint, + n_name varchar(25), + n_regionkey bigint, + n_comment varchar(152) +)`, + orders: `( + o_orderkey bigint, + o_custkey bigint, + o_orderstatus varchar(1), + o_totalprice decimal(15, 2), + o_orderdate date, + o_orderpriority varchar(15), + o_clerk varchar(15), + o_shippriority integer, + o_comment varchar(79) +)`, + part: `( + p_partkey bigint, + p_name varchar(55), + p_mfgr varchar(25), + p_brand varchar(10), + p_type varchar(25), + p_size integer, + p_container varchar(10), + p_retailprice decimal(15, 2), + p_comment varchar(23) +)`, + partsupp: `( + ps_partkey bigint, + ps_suppkey bigint, + ps_availqty integer, + ps_supplycost decimal(15, 2), + ps_comment varchar(199) +)`, + region: `( + r_regionkey bigint, + r_name varchar(25), + r_comment varchar(152) +)`, + supplier: `( + s_suppkey bigint, + s_name varchar(25), + s_address varchar(40), + s_nationkey bigint, + s_phone varchar(15), + s_acctbal decimal(15, 2), + s_comment varchar(101) +)` + }, + clickbench: { + hits: `( + WatchID bigint, + JavaEnable smallint, + Title varchar, + GoodEvent smallint, + EventTime bigint, + EventDate date, + CounterID integer, + ClientIP integer, + RegionID integer, + UserID bigint, + CounterClass smallint, + OS smallint, + UserAgent smallint, + URL varchar, + Referer varchar, + IsRefresh smallint, + RefererCategoryID smallint, + RefererRegionID integer, + URLCategoryID smallint, + URLRegionID integer, + ResolutionWidth smallint, + ResolutionHeight smallint, + ResolutionDepth smallint, + FlashMajor smallint, + FlashMinor smallint, + FlashMinor2 varchar, + NetMajor smallint, + NetMinor smallint, + UserAgentMajor smallint, + UserAgentMinor varchar(255), + CookieEnable smallint, + JavascriptEnable smallint, + IsMobile smallint, + MobilePhone smallint, + MobilePhoneModel varchar, + Params varchar, + IPNetworkID integer, + TraficSourceID smallint, + SearchEngineID smallint, + SearchPhrase varchar, + AdvEngineID smallint, + IsArtifical smallint, + WindowClientWidth smallint, + WindowClientHeight smallint, + ClientTimeZone smallint, + ClientEventTime bigint, + SilverlightVersion1 smallint, + SilverlightVersion2 smallint, + SilverlightVersion3 integer, + SilverlightVersion4 smallint, + PageCharset varchar, + CodeVersion integer, + IsLink smallint, + IsDownload smallint, + IsNotBounce smallint, + FUniqID bigint, + OriginalURL varchar, + HID integer, + IsOldCounter smallint, + IsEvent smallint, + IsParameter smallint, + DontCountHits smallint, + WithHash smallint, + HitColor varchar(1), + LocalEventTime bigint, + Age smallint, + Sex smallint, + Income smallint, + Interests smallint, + Robotness smallint, + RemoteIP integer, + WindowName integer, + OpenerName integer, + HistoryLength smallint, + BrowserLanguage varchar, + BrowserCountry varchar, + SocialNetwork varchar, + SocialAction varchar, + HTTPError smallint, + SendTiming integer, + DNSTiming integer, + ConnectTiming integer, + ResponseStartTiming integer, + ResponseEndTiming integer, + FetchTiming integer, + SocialSourceNetworkID smallint, + SocialSourcePage varchar, + ParamPrice bigint, + ParamOrderID varchar, + ParamCurrency varchar, + ParamCurrencyID smallint, + OpenstatServiceName varchar, + OpenstatCampaignID varchar, + OpenstatAdID varchar, + OpenstatSourceID varchar, + UTMSource varchar, + UTMMedium varchar, + UTMCampaign varchar, + UTMContent varchar, + UTMTerm varchar, + FromTag varchar, + HasGCLID smallint, + RefererHash bigint, + URLHash bigint, + CLID integer +)` + }, + tpcds: { + call_center: `( + cc_call_center_sk integer, + cc_call_center_id varchar, + cc_rec_start_date date, + cc_rec_end_date date, + cc_closed_date_sk double, + cc_open_date_sk integer, + cc_name varchar, + cc_class varchar, + cc_employees integer, + cc_sq_ft integer, + cc_hours varchar, + cc_manager varchar, + cc_mkt_id integer, + cc_mkt_class varchar, + cc_mkt_desc varchar, + cc_market_manager varchar, + cc_division integer, + cc_division_name varchar, + cc_company integer, + cc_company_name varchar, + cc_street_number varchar, + cc_street_name varchar, + cc_street_type varchar, + cc_suite_number varchar, + cc_city varchar, + cc_county varchar, + cc_state varchar, + cc_zip varchar, + cc_country varchar, + cc_gmt_offset decimal(3, 2), + cc_tax_percentage decimal(2, 2) +)`, + catalog_page: `( + cp_catalog_page_sk integer, + cp_catalog_page_id varchar, + cp_start_date_sk double, + cp_end_date_sk double, + cp_department varchar, + cp_catalog_number double, + cp_catalog_page_number double, + cp_description varchar, + cp_type varchar +)`, + catalog_returns: `( + cr_returned_date_sk integer, + cr_returned_time_sk integer, + cr_item_sk integer, + cr_refunded_customer_sk double, + cr_refunded_cdemo_sk double, + cr_refunded_hdemo_sk double, + cr_refunded_addr_sk double, + cr_returning_customer_sk double, + cr_returning_cdemo_sk double, + cr_returning_hdemo_sk double, + cr_returning_addr_sk double, + cr_call_center_sk double, + cr_catalog_page_sk double, + cr_ship_mode_sk double, + cr_warehouse_sk double, + cr_reason_sk double, + cr_order_number integer, + cr_return_quantity double, + cr_return_amount decimal(7, 2), + cr_return_tax decimal(6, 2), + cr_return_amt_inc_tax decimal(7, 2), + cr_fee decimal(5, 2), + cr_return_ship_cost decimal(7, 2), + cr_refunded_cash decimal(7, 2), + cr_reversed_charge decimal(7, 2), + cr_store_credit decimal(7, 2), + cr_net_loss decimal(7, 2) +)`, + catalog_sales: `( + cs_sold_date_sk double, + cs_sold_time_sk double, + cs_ship_date_sk double, + cs_bill_customer_sk double, + cs_bill_cdemo_sk double, + cs_bill_hdemo_sk double, + cs_bill_addr_sk double, + cs_ship_customer_sk double, + cs_ship_cdemo_sk double, + cs_ship_hdemo_sk double, + cs_ship_addr_sk double, + cs_call_center_sk double, + cs_catalog_page_sk double, + cs_ship_mode_sk double, + cs_warehouse_sk double, + cs_item_sk integer, + cs_promo_sk double, + cs_order_number integer, + cs_quantity double, + cs_wholesale_cost decimal(5, 2), + cs_list_price decimal(5, 2), + cs_sales_price decimal(5, 2), + cs_ext_discount_amt decimal(7, 2), + cs_ext_sales_price decimal(7, 2), + cs_ext_wholesale_cost decimal(7, 2), + cs_ext_list_price decimal(7, 2), + cs_ext_tax decimal(6, 2), + cs_coupon_amt decimal(7, 2), + cs_ext_ship_cost decimal(7, 2), + cs_net_paid decimal(7, 2), + cs_net_paid_inc_tax decimal(7, 2), + cs_net_paid_inc_ship decimal(7, 2), + cs_net_paid_inc_ship_tax decimal(7, 2), + cs_net_profit decimal(7, 2) +)`, + customer: `( + c_customer_sk integer, + c_customer_id varchar, + c_current_cdemo_sk double, + c_current_hdemo_sk double, + c_current_addr_sk integer, + c_first_shipto_date_sk double, + c_first_sales_date_sk double, + c_salutation varchar, + c_first_name varchar, + c_last_name varchar, + c_preferred_cust_flag varchar, + c_birth_day double, + c_birth_month double, + c_birth_year double, + c_birth_country varchar, + c_login varchar, + c_email_address varchar, + c_last_review_date double +)`, + customer_address: `( + ca_address_sk integer, + ca_address_id varchar, + ca_street_number varchar, + ca_street_name varchar, + ca_street_type varchar, + ca_suite_number varchar, + ca_city varchar, + ca_county varchar, + ca_state varchar, + ca_zip varchar, + ca_country varchar, + ca_gmt_offset decimal(4, 2), + ca_location_type varchar +)`, + customer_demographics: `( + cd_demo_sk integer, + cd_gender varchar, + cd_marital_status varchar, + cd_education_status varchar, + cd_purchase_estimate integer, + cd_credit_rating varchar, + cd_dep_count integer, + cd_dep_employed_count integer, + cd_dep_college_count integer +)`, + date_dim: `( + d_date_sk integer, + d_date_id varchar, + d_date date, + d_month_seq integer, + d_week_seq integer, + d_quarter_seq integer, + d_year integer, + d_dow integer, + d_moy integer, + d_dom integer, + d_qoy integer, + d_fy_year integer, + d_fy_quarter_seq integer, + d_fy_week_seq integer, + d_day_name varchar, + d_quarter_name varchar, + d_holiday varchar, + d_weekend varchar, + d_following_holiday varchar, + d_first_dom integer, + d_last_dom integer, + d_same_day_ly integer, + d_same_day_lq integer, + d_current_day varchar, + d_current_week varchar, + d_current_month varchar, + d_current_quarter varchar, + d_current_year varchar +)`, + household_demographics: `( + hd_demo_sk integer, + hd_income_band_sk integer, + hd_buy_potential varchar, + hd_dep_count integer, + hd_vehicle_count integer +)`, + income_band: `( + ib_income_band_sk integer, + ib_lower_bound integer, + ib_upper_bound integer +)`, + inventory: `( + inv_date_sk integer, + inv_item_sk integer, + inv_warehouse_sk integer, + inv_quantity_on_hand double +)`, + item: `( + i_item_sk integer, + i_item_id varchar, + i_rec_start_date date, + i_rec_end_date date, + i_item_desc varchar, + i_current_price decimal(4, 2), + i_wholesale_cost decimal(4, 2), + i_brand_id double, + i_brand varchar, + i_class_id double, + i_class varchar, + i_category_id double, + i_category varchar, + i_manufact_id double, + i_manufact varchar, + i_size varchar, + i_formulation varchar, + i_color varchar, + i_units varchar, + i_container varchar, + i_manager_id double, + i_product_name varchar +)`, + promotion: `( + p_promo_sk integer, + p_promo_id varchar, + p_start_date_sk double, + p_end_date_sk double, + p_item_sk double, + p_cost decimal(6, 2), + p_response_target double, + p_promo_name varchar, + p_channel_dmail varchar, + p_channel_email varchar, + p_channel_catalog varchar, + p_channel_tv varchar, + p_channel_radio varchar, + p_channel_press varchar, + p_channel_event varchar, + p_channel_demo varchar, + p_channel_details varchar, + p_purpose varchar, + p_discount_active varchar +)`, + reason: `( + r_reason_sk integer, + r_reason_id varchar, + r_reason_desc varchar +)`, + ship_mode: `( + sm_ship_mode_sk integer, + sm_ship_mode_id varchar, + sm_type varchar, + sm_code varchar, + sm_carrier varchar, + sm_contract varchar +)`, + store: `( + s_store_sk integer, + s_store_id varchar, + s_rec_start_date date, + s_rec_end_date date, + s_closed_date_sk double, + s_store_name varchar, + s_number_employees integer, + s_floor_space integer, + s_hours varchar, + s_manager varchar, + s_market_id integer, + s_geography_class varchar, + s_market_desc varchar, + s_market_manager varchar, + s_division_id integer, + s_division_name varchar, + s_company_id integer, + s_company_name varchar, + s_street_number varchar, + s_street_name varchar, + s_street_type varchar, + s_suite_number varchar, + s_city varchar, + s_county varchar, + s_state varchar, + s_zip varchar, + s_country varchar, + s_gmt_offset decimal(3, 2), + s_tax_precentage decimal(2, 2) +)`, + store_returns: `( + sr_returned_date_sk double, + sr_return_time_sk double, + sr_item_sk integer, + sr_customer_sk double, + sr_cdemo_sk double, + sr_hdemo_sk double, + sr_addr_sk double, + sr_store_sk double, + sr_reason_sk double, + sr_ticket_number integer, + sr_return_quantity double, + sr_return_amt decimal(7, 2), + sr_return_tax decimal(6, 2), + sr_return_amt_inc_tax decimal(7, 2), + sr_fee decimal(5, 2), + sr_return_ship_cost decimal(6, 2), + sr_refunded_cash decimal(7, 2), + sr_reversed_charge decimal(7, 2), + sr_store_credit decimal(7, 2), + sr_net_loss decimal(6, 2) +)`, + store_sales: `( + ss_sold_date_sk double, + ss_sold_time_sk double, + ss_item_sk integer, + ss_customer_sk double, + ss_cdemo_sk double, + ss_hdemo_sk double, + ss_addr_sk double, + ss_store_sk double, + ss_promo_sk double, + ss_ticket_number integer, + ss_quantity double, + ss_wholesale_cost decimal(5, 2), + ss_list_price decimal(5, 2), + ss_sales_price decimal(5, 2), + ss_ext_discount_amt decimal(7, 2), + ss_ext_sales_price decimal(7, 2), + ss_ext_wholesale_cost decimal(7, 2), + ss_ext_list_price decimal(7, 2), + ss_ext_tax decimal(6, 2), + ss_coupon_amt decimal(7, 2), + ss_net_paid decimal(7, 2), + ss_net_paid_inc_tax decimal(7, 2), + ss_net_profit decimal(6, 2) +)`, + time_dim: `( + t_time_sk integer, + t_time_id varchar, + t_time integer, + t_hour integer, + t_minute integer, + t_second integer, + t_am_pm varchar, + t_shift varchar, + t_sub_shift varchar, + t_meal_time varchar +)`, + warehouse: `( + w_warehouse_sk integer, + w_warehouse_id varchar, + w_warehouse_name varchar, + w_warehouse_sq_ft integer, + w_street_number varchar, + w_street_name varchar, + w_street_type varchar, + w_suite_number varchar, + w_city varchar, + w_county varchar, + w_state varchar, + w_zip varchar, + w_country varchar, + w_gmt_offset decimal(3, 2) +)`, + web_page: `( + wp_web_page_sk integer, + wp_web_page_id varchar, + wp_rec_start_date date, + wp_rec_end_date date, + wp_creation_date_sk integer, + wp_access_date_sk integer, + wp_autogen_flag varchar, + wp_customer_sk double, + wp_url varchar, + wp_type varchar, + wp_char_count integer, + wp_link_count integer, + wp_image_count integer, + wp_max_ad_count integer +)`, + web_returns: `( + wr_returned_date_sk double, + wr_returned_time_sk double, + wr_item_sk integer, + wr_refunded_customer_sk double, + wr_refunded_cdemo_sk double, + wr_refunded_hdemo_sk double, + wr_refunded_addr_sk double, + wr_returning_customer_sk double, + wr_returning_cdemo_sk double, + wr_returning_hdemo_sk double, + wr_returning_addr_sk double, + wr_web_page_sk double, + wr_reason_sk double, + wr_order_number integer, + wr_return_quantity double, + wr_return_amt decimal(7, 2), + wr_return_tax decimal(6, 2), + wr_return_amt_inc_tax decimal(7, 2), + wr_fee decimal(5, 2), + wr_return_ship_cost decimal(7, 2), + wr_refunded_cash decimal(7, 2), + wr_reversed_charge decimal(7, 2), + wr_account_credit decimal(7, 2), + wr_net_loss decimal(7, 2) +)`, + web_sales: `( + ws_sold_date_sk double, + ws_sold_time_sk double, + ws_ship_date_sk double, + ws_item_sk integer, + ws_bill_customer_sk double, + ws_bill_cdemo_sk double, + ws_bill_hdemo_sk double, + ws_bill_addr_sk double, + ws_ship_customer_sk double, + ws_ship_cdemo_sk double, + ws_ship_hdemo_sk double, + ws_ship_addr_sk double, + ws_web_page_sk double, + ws_web_site_sk double, + ws_ship_mode_sk double, + ws_warehouse_sk double, + ws_promo_sk double, + ws_order_number integer, + ws_quantity double, + ws_wholesale_cost decimal(5, 2), + ws_list_price decimal(5, 2), + ws_sales_price decimal(5, 2), + ws_ext_discount_amt decimal(7, 2), + ws_ext_sales_price decimal(7, 2), + ws_ext_wholesale_cost decimal(7, 2), + ws_ext_list_price decimal(7, 2), + ws_ext_tax decimal(6, 2), + ws_coupon_amt decimal(7, 2), + ws_ext_ship_cost decimal(7, 2), + ws_net_paid decimal(7, 2), + ws_net_paid_inc_tax decimal(7, 2), + ws_net_paid_inc_ship decimal(7, 2), + ws_net_paid_inc_ship_tax decimal(7, 2), + ws_net_profit decimal(7, 2) +)`, + web_site: `( + web_site_sk integer, + web_site_id varchar, + web_rec_start_date date, + web_rec_end_date date, + web_name varchar, + web_open_date_sk double, + web_close_date_sk double, + web_class varchar, + web_manager varchar, + web_mkt_id integer, + web_mkt_class varchar, + web_mkt_desc varchar, + web_market_manager varchar, + web_company_id integer, + web_company_name varchar, + web_street_number varchar, + web_street_name varchar, + web_street_type varchar, + web_suite_number varchar, + web_city varchar, + web_county varchar, + web_state varchar, + web_zip varchar, + web_country varchar, + web_gmt_offset decimal(3, 2), + web_tax_percentage decimal(2, 2) +)` } +} - return { rowCount }; - } - - async createTables(sf: number): Promise { - const schema = `tpch_sf${sf}`; - - // Create schema first - await this.executeSingleStatement(`CREATE SCHEMA IF NOT EXISTS hive.${schema} WITH (location = 's3://datafusion-distributed-benchmarks/tpch_sf${sf}/')`); - - // Create customer table - await this.executeSingleStatement(`DROP TABLE IF EXISTS hive.${schema}.customer`); - await this.executeSingleStatement(`CREATE TABLE hive.${schema}.customer - ( - c_custkey bigint, - c_name varchar(25), - c_address varchar(40), - c_nationkey bigint, - c_phone varchar(15), - c_acctbal decimal(15, 2), - c_mktsegment varchar(10), - c_comment varchar(117) - ) - WITH (external_location = 's3://datafusion-distributed-benchmarks/tpch_sf${sf}/customer/', format = 'PARQUET')`); - - // Create lineitem table - await this.executeSingleStatement(`DROP TABLE IF EXISTS hive.${schema}.lineitem`); - await this.executeSingleStatement(`CREATE TABLE hive.${schema}.lineitem - ( - l_orderkey bigint, - l_partkey bigint, - l_suppkey bigint, - l_linenumber integer, - l_quantity decimal(15, 2), - l_extendedprice decimal(15, 2), - l_discount decimal(15, 2), - l_tax decimal(15, 2), - l_returnflag varchar(1), - l_linestatus varchar(1), - l_shipdate date, - l_commitdate date, - l_receiptdate date, - l_shipinstruct varchar(25), - l_shipmode varchar(10), - l_comment varchar(44) - ) - WITH (external_location = 's3://datafusion-distributed-benchmarks/tpch_sf${sf}/lineitem/', format = 'PARQUET')`); - - // Create nation table - await this.executeSingleStatement(`DROP TABLE IF EXISTS hive.${schema}.nation`); - await this.executeSingleStatement(`CREATE TABLE hive.${schema}.nation - ( - n_nationkey bigint, - n_name varchar(25), - n_regionkey bigint, - n_comment varchar(152) - ) - WITH (external_location = 's3://datafusion-distributed-benchmarks/tpch_sf${sf}/nation/', format = 'PARQUET')`); - - // Create orders table - await this.executeSingleStatement(`DROP TABLE IF EXISTS hive.${schema}.orders`); - await this.executeSingleStatement(`CREATE TABLE hive.${schema}.orders - ( - o_orderkey bigint, - o_custkey bigint, - o_orderstatus varchar(1), - o_totalprice decimal(15, 2), - o_orderdate date, - o_orderpriority varchar(15), - o_clerk varchar(15), - o_shippriority integer, - o_comment varchar(79) - ) - WITH (external_location = 's3://datafusion-distributed-benchmarks/tpch_sf${sf}/orders/', format = 'PARQUET')`); - - // Create part table - await this.executeSingleStatement(`DROP TABLE IF EXISTS hive.${schema}.part`); - await this.executeSingleStatement(`CREATE TABLE hive.${schema}.part - ( - p_partkey bigint, - p_name varchar(55), - p_mfgr varchar(25), - p_brand varchar(10), - p_type varchar(25), - p_size integer, - p_container varchar(10), - p_retailprice decimal(15, 2), - p_comment varchar(23) - ) - WITH (external_location = 's3://datafusion-distributed-benchmarks/tpch_sf${sf}/part/', format = 'PARQUET')`); - - // Create partsupp table - await this.executeSingleStatement(`DROP TABLE IF EXISTS hive.${schema}.partsupp`); - await this.executeSingleStatement(`CREATE TABLE hive.${schema}.partsupp - ( - ps_partkey bigint, - ps_suppkey bigint, - ps_availqty integer, - ps_supplycost decimal(15, 2), - ps_comment varchar(199) - ) - WITH (external_location = 's3://datafusion-distributed-benchmarks/tpch_sf${sf}/partsupp/', format = 'PARQUET')`); - - // Create region table - await this.executeSingleStatement(`DROP TABLE IF EXISTS hive.${schema}.region`); - await this.executeSingleStatement(`CREATE TABLE hive.${schema}.region - ( - r_regionkey bigint, - r_name varchar(25), - r_comment varchar(152) - ) - WITH (external_location = 's3://datafusion-distributed-benchmarks/tpch_sf${sf}/region/', format = 'PARQUET')`); - - // Create supplier table - await this.executeSingleStatement(`DROP TABLE IF EXISTS hive.${schema}.supplier`); - await this.executeSingleStatement(`CREATE TABLE hive.${schema}.supplier - ( - s_suppkey bigint, - s_name varchar(25), - s_address varchar(40), - s_nationkey bigint, - s_phone varchar(15), - s_acctbal decimal(15, 2), - s_comment varchar(101) - ) - WITH (external_location = 's3://datafusion-distributed-benchmarks/tpch_sf${sf}/supplier/', format = 'PARQUET')`); - } +function getSchema(table: TableSpec): string { + const tableSchema = SCHEMAS[table.schema.split("_")[0]]?.[table.name] + if (!tableSchema) { + throw new Error(`Could not find table ${table.name} in schema ${table.schema}`) + } + return tableSchema } main() - .catch(err => { - console.error(err) - process.exit(1) - }) + .catch(err => { + console.error(err) + process.exit(1) + }) diff --git a/benchmarks/cdk/bin/worker.rs b/benchmarks/cdk/bin/worker.rs index ab3e398d..45bdfbc4 100644 --- a/benchmarks/cdk/bin/worker.rs +++ b/benchmarks/cdk/bin/worker.rs @@ -6,11 +6,11 @@ use datafusion::common::DataFusionError; use datafusion::common::instant::Instant; use datafusion::common::runtime::SpawnedTask; use datafusion::execution::SessionStateBuilder; +use datafusion::execution::runtime_env::RuntimeEnv; use datafusion::physical_plan::execute_stream; use datafusion::prelude::SessionContext; use datafusion_distributed::{ - DistributedExt, DistributedPhysicalOptimizerRule, Worker, WorkerQueryContext, WorkerResolver, - display_plan_ascii, + DistributedExt, DistributedPhysicalOptimizerRule, Worker, WorkerResolver, display_plan_ascii, }; use futures::{StreamExt, TryFutureExt}; use log::{error, info, warn}; @@ -64,19 +64,18 @@ async fn main() -> Result<(), Box> { .with_bucket_name(s3_url.host().unwrap().to_string()) .build()?, ); + let runtime_env = Arc::new(RuntimeEnv::default()); + runtime_env.register_object_store(&s3_url, s3); + let state = SessionStateBuilder::new() .with_default_features() - .with_object_store(&s3_url, Arc::clone(&s3) as _) + .with_runtime_env(Arc::clone(&runtime_env)) .with_distributed_worker_resolver(Ec2WorkerResolver::new()) .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) .build(); let ctx = SessionContext::from(state); - let arrow_flight_endpoint = Worker::from_session_builder(move |ctx: WorkerQueryContext| { - let s3 = s3.clone(); - let s3_url = s3_url.clone(); - async move { Ok(ctx.builder.with_object_store(&s3_url, s3).build()) } - }); + let worker = Worker::default().with_runtime_env(runtime_env); let http_server = axum::serve( listener, Router::new().route( @@ -137,7 +136,7 @@ async fn main() -> Result<(), Box> { ), ); let grpc_server = Server::builder() - .add_service(arrow_flight_endpoint.into_flight_server()) + .add_service(worker.into_flight_server()) .serve(WORKER_ADDR.parse()?); info!("Started listener HTTP server in {LISTENER_ADDR}"); diff --git a/src/flight_service/worker.rs b/src/flight_service/worker.rs index ecd7f8b1..28355407 100644 --- a/src/flight_service/worker.rs +++ b/src/flight_service/worker.rs @@ -57,6 +57,13 @@ impl Worker { } } + /// Sets a [RuntimeEnv] to be used in all the queries this [Worker] will handle during + /// its lifetime. + pub fn with_runtime_env(mut self, runtime_env: Arc) -> Self { + self.runtime = runtime_env; + self + } + /// Adds a callback for when an [ExecutionPlan] is received in the `do_get` call. /// /// The callback takes the plan and returns another plan that must be either the same, From 9509b85d4ea8cde303f2672fa0b5dd52ba4d4903 Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Tue, 6 Jan 2026 20:00:11 +0100 Subject: [PATCH 13/27] Improve docs and add custom execution plan example (#277) * Support any dataset in benchmarks * Add TPC-DS schemas * Improve docs * Let Claude improve wording * Add custom_execution_plan.rs example * Link to custom_execution_plan.rs example * Respond to PR feedback --- docs/source/_static/images/concepts.png | 3 + docs/source/index.rst | 5 +- docs/source/user-guide/channel-resolver.md | 36 +- docs/source/user-guide/concepts.md | 62 +-- docs/source/user-guide/getting-started.md | 108 +++-- .../how-a-distributed-plan-is-built.md | 29 +- docs/source/user-guide/index.md | 16 - docs/source/user-guide/task-estimator.md | 22 +- docs/source/user-guide/worker-resolver.md | 27 +- docs/source/user-guide/worker.md | 17 +- examples/custom_execution_plan.md | 148 +++++++ examples/custom_execution_plan.rs | 408 ++++++++++++++++++ 12 files changed, 719 insertions(+), 162 deletions(-) create mode 100644 docs/source/_static/images/concepts.png delete mode 100644 docs/source/user-guide/index.md create mode 100644 examples/custom_execution_plan.md create mode 100644 examples/custom_execution_plan.rs diff --git a/docs/source/_static/images/concepts.png b/docs/source/_static/images/concepts.png new file mode 100644 index 00000000..60c89f0c --- /dev/null +++ b/docs/source/_static/images/concepts.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3714f57b3815db2004a1f6445298256b3f63cac9882e5186f901357e688fcbee +size 262763 diff --git a/docs/source/index.rst b/docs/source/index.rst index 1cea0739..ea80de49 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -13,13 +13,12 @@ how to contribute changes to the library yourself. :maxdepth: 2 :caption: User Guide - user-guide/index + user-guide/concepts user-guide/getting-started + user-guide/worker user-guide/worker-resolver - user-guide/arrow-flight-endpoint user-guide/channel-resolver user-guide/task-estimator - user-guide/concepts user-guide/how-a-distributed-plan-is-built .. _toc.contributor-guide: diff --git a/docs/source/user-guide/channel-resolver.md b/docs/source/user-guide/channel-resolver.md index c984459b..749cabf0 100644 --- a/docs/source/user-guide/channel-resolver.md +++ b/docs/source/user-guide/channel-resolver.md @@ -1,23 +1,23 @@ # Building a ChannelResolver -This trait is optional, there's a sane default already in place that should work for most simple use cases. +This trait is optional—a sensible default implementation exists that handles most use cases. -A `ChannelResolver` tells Distributed DataFusion how to build an Arrow Flight client baked by a -[tonic](https://github.com/hyperium/tonic) channel given a worker URL. +The `ChannelResolver` trait controls how Distributed DataFusion builds Arrow Flight clients backed by +[Tonic](https://github.com/hyperium/tonic) channels for worker URLs. -There's a default implementation that simply connects to the given URL, builds the Arrow Flight client instance, -and caches it so that the same client instance gets reused for a query to the same URL. +The default implementation connects to each URL, builds an Arrow Flight client, and caches it for reuse on +subsequent requests to the same URL. -However, you might want to provide your own implementation. For that you need to take into account the following -points: +## Providing your own ChannelResolver + +For providing your own implementation, you'll need to take into account the following points: - You will need to provide your own implementation in two places: - - in the `SessionContext` that handles your queries. + - in the `SessionContext` that first initiates and plans your queries. - while instantiating the `Worker` with the `from_session_builder()` constructor. -- If you decide to build it from scratch, make sure that Arrow Flight clients are reused across - request rather than always building new ones. -- You can use this library's `DefaultChannelResolver` as a backbone for your own implementation. - If you do that, channel caching will be automatically managed for you. +- If building from scratch, ensure Arrow Flight clients are reused across requests rather than recreated each time. +- You can extend `DefaultChannelResolver` as a foundation for custom implementations. This automatically handles + gRPC channel reuse. ```rust #[derive(Clone)] @@ -25,15 +25,19 @@ struct CustomChannelResolver; #[async_trait] impl ChannelResolver for CustomChannelResolver { - async fn get_flight_client_for_url(&self, url: &Url) -> Result, DataFusionError> { - // Build a custom FlightServiceClient wrapped with tower layers or something similar. + async fn get_flight_client_for_url( + &self, + url: &Url, + ) -> Result, DataFusionError> { + // Build a custom FlightServiceClient wrapped with tower + // layers or something similar. todo!() } } async fn main() { - // Make sure you build just one for the whole lifetime of your application, as it needs to be able to - // reuse Arrow Flight client instances across different queries. + // Build a single instance for your application's lifetime + // to enable Arrow Flight client reuse across queries. let channel_resolver = CustomChannelResolver; let state = SessionStateBuilder::new() diff --git a/docs/source/user-guide/concepts.md b/docs/source/user-guide/concepts.md index d5e81007..88cbd0aa 100644 --- a/docs/source/user-guide/concepts.md +++ b/docs/source/user-guide/concepts.md @@ -1,25 +1,50 @@ # Concepts -This library contains a set of extensions to DataFusion that allow you to run distributed queries. +This library is a collection of DataFusion extensions that enable distributed query execution. You can think of +it as normal DataFusion, with the addition that some nodes are capable of streaming data over the network using +Arrow Flight instead of through in-memory communication. -These are some terms you should be familiar with before getting started: +Key terminology: - `Stage`: a portion of the plan separated by a network boundary from other parts of the plan. A plan contains - one or more stages. + one or more stages, each separated by network boundaries. - `Task`: a unit of work in a stage that executes the inner plan in parallel to other tasks within the stage. Each task - in a stage is executed by a different worker. -- `Worker`: a physical machine listening to serialized execution plans over an Arrow Flight interface. -- `Network boundary`: a node in the plan that streams data from a network interface rather than directly from its - children. Implemented as DataFusion `ExecutionPlan`s: `NeworkShuffle` and `NetworkCoalesce`. -- `Subplan`: a slice of the overall plan. Each stage will execute a subplan of the overall plan. + in a stage executes a structurally identical plan in different worker, passing a `task_index` as a contextual value + for making choices about what data should be returned. +- `Network Boundary`: a node in the plan that streams data from a network interface rather than directly from its + children nodes. +- `Worker`: a physical machine listening to serialized execution plans over an Arrow Flight interface. A task is + executed by exactly one worker, but one worker executes many tasks concurrently. + +![concepts.png](../_static/images/concepts.png) + +You'll see these concepts mentioned extensively across the documentation and the code itself. + +# Public API + +Some other more tangible concepts are the structs and traits exposed publicly, the most important are: ## [DistributedPhysicalOptimizerRule](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/src/distributed_planner/distributed_physical_optimizer_rule.rs) -This is a physical optimizer rule that converts a single-node DataFusion query into a distributed query. It reads +A physical optimizer rule that transforms single-node DataFusion query plans into distributed query plans. It reads a fully formed physical plan and injects the appropriate nodes to execute the query in a distributed fashion. -It builds the distributed plan from bottom to top, and based on the present nodes in the original plan, -it will inject network boundaries in the appropriate places. +It builds the distributed plan from bottom to top, injecting network boundaries at appropriate locations based on +the nodes present in the original plan. + +## [Worker](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/src/flight_service/worker.rs) + +Arrow Flight server implementation that integrates with the Tonic ecosystem and listens to serialized plans that get +executed over the wire. + +Users are expected to build these and spawn them in ports so that the network boundary nodes can reach them. + +## [WorkerResolver](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/src/networking/worker_resolver.rs) + +Determines the available workers in the Distributed DataFusion cluster by returning their URLs. + +Different organizations have different networking requirements—from Kubernetes deployments to cloud provider +solutions. This trait allows Distributed DataFusion to adapt to various scenarios. ## [TaskEstimator](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/src/distributed_planner/task_estimator.rs) @@ -41,22 +66,7 @@ you are in, you might want to return a different set of data. For example, if you are on the task with index 0 of a 3-task stage, you might want to return only the first 1/3 of the data. If you are on the task with index 2, you might want to return the last 1/3 of the data, and so on. -## [WorkerResolver](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/src/networking/worker_resolver.rs) - -Establishes the number of workers available in the distributed DataFusion cluster by returning their URLs. - -Different organizations have different needs regarding networking. Some might be using Kubernetes, some other might -be using a cloud provider solution, and this trait allows adapting Distributed DataFusion to the different scenarios. - ## [ChannelResolver](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/src/networking/channel_resolver.rs) Optional extension trait that allows to customize how connections are established to workers. Given one of the URLs returned by the `WorkerResolver`, it builds an Arrow Flight client ready for serving queries. - -## [NetworkBoundary](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/src/distributed_planner/network_boundary.rs) - -A network boundary is a node that, instead of pulling data from its children by executing them, serializes them -and sends them over the wire so that they are executed on a remote worker. - -As a user of distributed DataFusion, you will not need to interact with this trait directly, but you should know -that different implementations of this trait will be injected into your plans so that queries get distributed. diff --git a/docs/source/user-guide/getting-started.md b/docs/source/user-guide/getting-started.md index 0682180c..33f5fa02 100644 --- a/docs/source/user-guide/getting-started.md +++ b/docs/source/user-guide/getting-started.md @@ -1,28 +1,36 @@ # Getting Started -Rather than being opinionated about your setup and how you serve queries to users, -Distributed DataFusion allows you to plug in your own networking stack and spawn your own gRPC servers that act as -workers in the cluster. +Think of this library as vanilla DataFusion, except that certain nodes execute their children on remote +machines and retrieve data via the Arrow Flight protocol. + +This library aims to provide an experience as close as possible to vanilla DataFusion. + +## How to use Distributed DataFusion + +Rather than imposing constraints on your infrastructure or query serving patterns, Distributed DataFusion +allows you to plug in your own networking stack and spawn your own gRPC servers that act as workers in the cluster. This project heavily relies on the [Tonic](https://github.com/hyperium/tonic) ecosystem for the networking layer. Users of this library are responsible for building their own Tonic server, adding the Arrow Flight distributed -DataFusion service to it and spawning it on a port so that it can be reached by other workers in the cluster. +DataFusion service to it and spawning it on a port so that it can be reached by other workers in the cluster. A very +basic example of this would be: -For a basic setup, all you need to do is to enrich your DataFusion `SessionStateBuilder` with the tools this project -ships: +```rust +#[tokio::main] +async fn main() -> Result<(), Box> { + let worker = Worker::default(); -```rs - let state = SessionStateBuilder::new() -+ .with_distributed_worker_resolver(my_custom_worker_resolver) -+ .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) - .build(); -``` + Server::builder() + .add_service(worker.into_flight_server()) + .serve(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8000)) + .await?; -And the `my_custom_worker_resolver` variable should be an implementation of -the [WorkerResolverResolver](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/src/networking/worker_resolver.rs) -trait, which tells Distributed DataFusion how to connect to other workers in the cluster. + Ok(()) +} +``` -A very basic example of such an implementation that resolves workers in the localhost machine is: +Distributed DataFusion requires knowledge of worker locations. Implement the `WorkerResolver` trait to provide +this information. Here is a simple example of what this would look like with localhost workers: ```rust #[derive(Clone)] @@ -30,59 +38,49 @@ struct LocalhostWorkerResolver { ports: Vec, } -#[async_trait] -impl ChannelResolver for LocalhostChannelResolver { +impl WorkerResolver for LocalhostWorkerResolver { fn get_urls(&self) -> Result, DataFusionError> { - Ok(self.ports.iter().map(|port| Url::parse(&format!("http://localhost:{port}")).unwrap()).collect()) + Ok(self + .ports + .iter() + .map(|port| Url::parse(&format!("http://localhost:{port}")).unwrap()) + .collect()) } } ``` -> NOTE: This example is not production-ready and is meant to showcase the basic concepts of the library. - -This `WorkerResolver` implementation should resolve URLs of Distributed DataFusion workers, and it's -also the user of this library's responsibility to spawn a Tonic server that exposes the worker as an Arrow Flight -service using Tonic. +Register both the `WorkerResolver` implementation and the `DistributedPhysicalOptimizerRule` in DataFusion's +`SessionStateBuilder` to enable distributed query planning: -A basic example of such a server is: - -```rust -#[tokio::main] -async fn main() -> Result<(), Box> { - let endpoint = Worker::default(); - - Server::builder() - .add_service(endpoint.into_flight_server()) - .serve(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8000)) - .await?; - - Ok(()) +```rs +let localhost_worker_resolver = LocalhostWorkerResolver { + ports: vec![8000, 8001, 8002] } -``` -## Next steps +let state = SessionStateBuilder::new() + .with_distributed_worker_resolver(localhost_worker_resolver) + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + .build(); -The next two sections of this guide will walk you through tailoring the library's traits to your own needs: +let ctx = SessionContext::from(state); +``` -- [Build your own WorkerResolver](worker-resolver.md) -- [Build your own ChannelResolver](channel-resolver.md) -- [Build your own TaskEstimator](task-estimator.md) -- [Build your own distributed DataFusion Worker](worker.md) +This will leave a DataFusion `SessionContext` ready for executing distributed queries. -Here are some other resources in the codebase: +> NOTE: This example is not production-ready and is meant to showcase the basic concepts of the library. -- [In-memory cluster example](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/examples/in_memory.md) -- [Localhost cluster example](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/examples/localhost.md) +## Next steps -A more advanced example can be found in the benchmarks that use a cluster of distributed DataFusion workers -deployed on AWS EC2 machines: +Depending on your needs, your setup can get more complicated, for example: -- [AWS EC2 based cluster example](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/benchmarks/cdk/bin/worker.rs) +- You may want to resolve worker URLs dynamically using the Kubernetes API. +- You may want to wrap the Arrow Flight clients that connect workers with an observability layer. +- You may want to be able to execute your own custom ExecutionPlans in a distributed manner. +- etc... -Each feature in the project is showcased and tested in its own isolated integration test, so it's recommended to -review those for a better understanding of how specific features work: +To learn how to do all that, it's recommended to: -- [Pass your own ConfigExtension implementations across network boundaries](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/tests/custom_config_extension.rs) -- [Provide custom protobuf codecs for your own nodes](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/tests/custom_extension_codec.rs) -- Provide a custom TaskEstimator for controlling the amount of parallelism (coming soon) +- [Continue reading this guide](worker.md) +- [Look at examples in the project](https://github.com/datafusion-contrib/datafusion-distributed/tree/main/examples) +- [Look at the integration tests for finer grained examples](https://github.com/datafusion-contrib/datafusion-distributed/tree/main/tests) diff --git a/docs/source/user-guide/how-a-distributed-plan-is-built.md b/docs/source/user-guide/how-a-distributed-plan-is-built.md index c872fd53..7b1efbd5 100644 --- a/docs/source/user-guide/how-a-distributed-plan-is-built.md +++ b/docs/source/user-guide/how-a-distributed-plan-is-built.md @@ -1,6 +1,6 @@ # How a Distributed Plan is Built -This page walks through the steps the distributed DataFusion planner takes to build a distributed query plan. +This page walks through how the distributed DataFusion planner transforms a query into a distributed execution plan. Everything starts with a simple single-node plan, for example: @@ -29,11 +29,14 @@ The first step is to split the leaf node into different tasks: Each task will handle a different non-overlapping piece of data. The number of tasks that will be used for executing leaf nodes is determined by a `TaskEstimator` implementation. -There is one default implementation for a file-based `DataSourceExec`, but since a `DataSourceExec` in DataFusion can -be anything the user wants to implement, it is the user who should also provide a custom `TaskEstimator` if they have -a custom `DataSourceExec`. +A default implementation exists for file-based `DataSourceExec` nodes. However, since `DataSourceExec` can be +customized to represent any data source, users with custom implementations should also provide a corresponding +`TaskEstimator`. -In the case above, a `TaskEstimator` decided to use four tasks for the leaf node. +In the case above, a `TaskEstimator` decided to use four tasks for the leaf node. Note that even if we are distributing +the data across different tasks, each task will also distribute its data across partitions using the vanilla DataFusion +partitioning mechanism. A partition is a split of data processed by a single thread on a single machine, whereas a +task is a split of data processed by an entire machine within a cluster. After that, we can continue reconstructing the plan: @@ -46,12 +49,11 @@ Let's keep constructing the plan: ![img_4.png](../_static/images/img_4.png) -Things change at this point: a `RepartitionExec` implies that we want to repartition data so that each partition handles -a non-overlapping set of the grouping keys of the ongoing aggregation. +At this point, the plan encounters a `RepartitionExec` node, which requires repartitioning data so each partition +handles a non-overlapping subset of grouping keys for the aggregation. -If a `RepartitionExec` in a single-node context redistributes data across threads on the same machine, a -`RepartitionExec` in a distributed context redistributes data across threads on different machines. This means that we -need to perform a shuffle over the network. +While `RepartitionExec` redistributes data across threads on a single machine in vanilla DataFusion, it redistributes +data across threads on different machines in the distributed context—requiring a network shuffle. As we are about to send data over the network, it's convenient to coalesce smaller batches into larger ones to avoid the overhead of sending many small messages, and instead send fewer but larger messages: @@ -80,9 +82,10 @@ The rest of the plan can be formed as normal: ![img_7.png](../_static/images/img_7.png) -There's just one last step: the head of the plan is currently spread across two different machines, but we want it -on one. In the same way that vanilla DataFusion coalesces all partitions into one in the head node for the user, we also -need to do that, but not only across partitions on a single machine, but across tasks on different machines. +One final step remains: the plan's head is currently distributed across two machines, but the final result must be +consolidated on a single node. In the same way that vanilla DataFusion coalesces all partitions into one in the head +node for the user, we also need to do that, but not only across partitions on a single machine, but across tasks on +different machines. For that, the `NetworkCoalesceExec` network boundary is introduced: it coalesces P partitions across N tasks into N*P partitions in one task. This does not imply repartitioning, or shuffling, or anything like that. The partitions diff --git a/docs/source/user-guide/index.md b/docs/source/user-guide/index.md deleted file mode 100644 index f4d4022d..00000000 --- a/docs/source/user-guide/index.md +++ /dev/null @@ -1,16 +0,0 @@ -# Index - -Distributed DataFusion is a library that brings distributed capabilities to DataFusion. -It provides a set of execution plans, optimization rules, configuration extensions, and new traits -to enable distributed execution. - -This user guide will walk you through using the tools in this project to set up -your own distributed DataFusion cluster. - -- [Concepts](concepts.md) -- [Getting Started](getting-started.md) -- [Building a WorkerResolver](worker-resolver.md) -- [Spawning a Worker](worker.md) -- [Building a ChannelResolver](channel-resolver.md) -- [Building a TaskEstimator](task-estimator.md) -- [How a distributed plan is built](how-a-distributed-plan-is-built.md) diff --git a/docs/source/user-guide/task-estimator.md b/docs/source/user-guide/task-estimator.md index 77eaacb1..b2d42f4d 100644 --- a/docs/source/user-guide/task-estimator.md +++ b/docs/source/user-guide/task-estimator.md @@ -1,15 +1,14 @@ # Building a TaskEstimator -A `TaskEstimator` is a trait that tells the distributed planner how many distributed tasks should be used in the -different stages of the plan. +The `TaskEstimator` trait controls how many distributed tasks the planner allocates to each stage of the query plan. -The number of tasks is assigned to the different stages in a bottom-to-top fashion. You can refer the +The number of tasks is assigned to the different stages in a bottom-up fashion. You can refer to the [Plan Annotation docs](https://github.com/datafusion-contrib/datafusion-distributed/blob/75b4e73e9052c6596b9d1744ce2bdfa6cbc010d3/src/distributed_planner/plan_annotator.rs) for an explanation on how this works. A `TaskEstimator` is what hints this process how many tasks should be used. -There is a default implementation that acts on `DataSourceExec` nodes baked by `FileScanConfig`s, however, as a user, -you may want to provide your own `TaskEstimator` implementation for your own `ExecutionPlan`s. +While a default implementation exists for file-based `DataSourceExec` nodes (those backed by `FileScanConfig`), you +can provide custom `TaskEstimator` implementations for your own `ExecutionPlan` types. ## Providing your own TaskEstimator @@ -19,12 +18,13 @@ Providing a `TaskEstimator` allows you to do two things: 2. Tell the distributed planner how to "scale up" your `ExecutionPlan` in order to account for it running in multiple distributed tasks. -Note that if your custom nodes are going to run distributed, you need to account for it at execution time. -If you build a `TaskEstimator` that tells the distributed planner that your node should run in N tasks, then -you need to react to the presence of a -[DistributedTaskCtx](https://github.com/datafusion-contrib/datafusion-distributed/blob/75b4e73e9052c6596b9d1744ce2bdfa6cbc010d3/src/stage.rs#L137-L137) -during execution. +If your custom nodes will execute in a distributed manner, you must handle this during execution. When your +`TaskEstimator` specifies N tasks for a node, your execution logic must respond to the +[DistributedTaskContext](https://github.com/datafusion-contrib/datafusion-distributed/blob/75b4e73e9052c6596b9d1744ce2bdfa6cbc010d3/src/stage.rs#L137-L137) +present in DataFusion's `TaskContext` to determine which subset of data this task should process. There's an example of how to do that in the `examples/` folder: -- TODO: link to example +- [custom_execution_plan.rs](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/examples/custom_execution_plan.rs) - + A complete example showing how to implement a custom execution plan (`numbers(start, end)` table function) + that works with distributed DataFusion, including a custom codec and TaskEstimator. diff --git a/docs/source/user-guide/worker-resolver.md b/docs/source/user-guide/worker-resolver.md index 8af97362..a17e5915 100644 --- a/docs/source/user-guide/worker-resolver.md +++ b/docs/source/user-guide/worker-resolver.md @@ -1,13 +1,12 @@ -# Building a WorkerResolver +# Implementing a WorkerResolver -A `WorkerResolver` tells distributed DataFusion the location (URLs) of your worker nodes. This information is -used in two different places: +The `WorkerResolver` trait provides Distributed DataFusion with the locations (URLs) of your worker nodes. This +information is used in two different places: -1. For planning: during distributed planning, the amount of worker nodes available (essentially, `Vec`.length()), - is used for determined how to scale up the plan. The distributed planner will not use more tasks per stage than the - amount of workers the cluster has. -2. Right before execution: each task in a distributed plan contains slots that the `DistributedExec` node needs - to fill right before execution (with the URLs of the workers as updated as possible). +1. **During planning**: The number of available workers (i.e., `Vec.len()`) determines how the plan scales. + The planner will not allocate more tasks per stage than available workers. +2. **Before execution**: Each task in a distributed plan needs its worker URL assignment populated right before + execution. You need to pass your own `WorkerResolver` to DataFusion's `SessionStateBuilder` so that it's available in the `SesionContext`: @@ -30,11 +29,12 @@ async fn main() { } ``` -> NOTE: It's not necessary to pass this to the Worker session builder. +> NOTE: It's not necessary to pass a WorkerResolver to the Worker session builder, it's just necessary on the +> SessionState that initiates and plans the query. ## Static WorkerResolver -This is the simplest thing you can do, although it will not fit some common use cases. An example of this can be +This is the simplest approach, though it doesn't accommodate dynamic worker discovery. An example of this can be seen in the [localhost_worker.rs](https://github.com/datafusion-contrib/datafusion-distributed/blob/fad9fa222d65b7d2ddae92fbc20082b5c434e4ff/examples/localhost_run.rs) example: @@ -64,9 +64,10 @@ provider for hosting your Distributed DataFusion workers. It's up to you to decide how the URLs should be resolved. One important implementation note is: -- Retrieving the worker URLs needs to be synchronous (as planning is synchronous), and therefore, you'll likely - need to spawn background tasks that periodically refresh the list of available URLs. +- Since planning is synchronous, `get_urls()` must return immediately. For dynamic worker discovery, spawn + background tasks that periodically refresh the worker URL list, storing results that `get_urls()` can access + synchronously. A good example can be found -in https://github.com/datafusion-contrib/datafusion-distributed/blob/main/benchmarks/cdk/bin/worker.rs, +in [benchmarks/cdk/bin/worker.rs](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/benchmarks/cdk/bin/worker.rs), where a cluster of AWS EC2 machines is discovered identified by tags with the AWS Rust SDK. \ No newline at end of file diff --git a/docs/source/user-guide/worker.md b/docs/source/user-guide/worker.md index 573d36c4..e57bc329 100644 --- a/docs/source/user-guide/worker.md +++ b/docs/source/user-guide/worker.md @@ -1,7 +1,7 @@ # Spawn a Worker -A `Worker` is a gRPC server that implements the Arrow Flight protocol for distributed query execution. -Worker nodes run these endpoints to receive execution plans, execute them, and stream results back. +The `Worker` is a gRPC server implementing the Arrow Flight protocol for distributed query execution. Worker nodes +run these endpoints to receive execution plans, execute them, and stream results back. ## Overview @@ -14,7 +14,7 @@ The `Worker` is the core worker component in Distributed DataFusion. It: ## Launching the Arrow Flight server -The `Default` implementation of the `Worker` should satisfy most basic use cases +The default `Worker` implementation satisfies most basic use cases: ```rust use datafusion_distributed::Worker; @@ -31,8 +31,7 @@ async fn main() { } ``` -If you are using DataFusion, though, it's very likely that you have your own custom UDFs, execution nodes, config -options, etc... +However, most DataFusion deployments include custom UDFs, execution nodes, or configuration options. You'll need to tell the `Worker` how to build your DataFusion sessions: @@ -56,7 +55,7 @@ async fn main() { } ``` -### WorkerSessionBuilder +## WorkerSessionBuilder The `WorkerSessionBuilder` is a closure or type that implements: @@ -73,7 +72,8 @@ pub trait WorkerSessionBuilder { It receives a `WorkerQueryContext` containing: - `SessionStateBuilder`: A pre-populated session state builder in which you can inject your custom stuff -- `headers`: HTTP headers from the incoming request (useful for passing metadata) +- `headers`: HTTP headers from the incoming request (useful for passing metadata like authentication tokens or + configuration) ## Serving the Endpoint @@ -94,5 +94,4 @@ async fn main() { } ``` -The `into_flight_server()` method wraps the endpoint in a `FlightServiceServer` with high message size limits ( -`usize::MAX`) to avoid chunking overhead for internal communication. +The `into_flight_server()` method builds a `FlightServiceServer` ready to be added as a Tonic service. diff --git a/examples/custom_execution_plan.md b/examples/custom_execution_plan.md new file mode 100644 index 00000000..49f2366a --- /dev/null +++ b/examples/custom_execution_plan.md @@ -0,0 +1,148 @@ +# Custom Execution Plan Example + +Demonstrates how to create a distributed custom execution plan with a `numbers(start, end)` table function. + +## Components + +**NumbersTableFunction** – Table function callable in SQL: `SELECT * FROM numbers(1, 100)` + +**NumbersExec** – Execution plan with `ranges_per_task: Vec>` storing one range per task. +Uses `DistributedTaskContext` to determine which range to generate. + +**NumbersExecCodec** – Protobuf-based serialization implementing `PhysicalExtensionCodec`. +Must be registered in the `SessionStateBuilder` that initiates the query as well as the one used by `Worker`s. + +**NumbersTaskEstimator** – Controls distributed parallelism: + +- `task_estimation()` - Returns how many tasks needed based on range size and config +- `scale_up_leaf_node()` - Splits single range of numbers into N per-task ranges + +**NumbersConfig** – Custom config extension for controlling distributed parallelism (`numbers_per_task: usize`) + +## Usage + +This example imports the `InMemoryWorkerResolver` and the `InMemoryChannelResolver` used during integration testing +of this project, so it needs the `--features integration` flag on. + +This example demonstrates how the bigger the range of numbers is queried, the more tasks are used in executing +the query, for example: + +```bash +cargo run \ + --features integration \ + --example custom_execution_plan \ + "SELECT DISTINCT number FROM numbers(0, 10) ORDER BY number" \ + --show-distributed-plan +``` + +``` +SortPreservingMergeExec: [number@0 ASC NULLS LAST] + SortExec: expr=[number@0 ASC NULLS LAST], preserve_partitioning=[true] + AggregateExec: mode=FinalPartitioned, gby=[number@0 as number], aggr=[] + CoalesceBatchesExec: target_batch_size=8192 + RepartitionExec: partitioning=Hash([number@0], 16), input_partitions=16 + AggregateExec: mode=Partial, gby=[number@0 as number], aggr=[] + RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 + CooperativeExec + NumbersExec: t0:[0-10) +``` + +This will print a non-distributed plan, as the range of numbers we are querying (`numbers(0, 10)`) is small. + +The config parameter `numbers.numbers_per_task` is the one that controls how many distributed tasks are used in the +query, and it's default value is `10`, so querying 10 numbers will not distribute the plan. + +However, if we try to query 11 numbers: + +```bash +cargo run \ + --features integration \ + --example custom_execution_plan \ + "SELECT DISTINCT number FROM numbers(0, 11) ORDER BY number" \ + --show-distributed-plan +``` + +``` +┌───── DistributedExec ── Tasks: t0:[p0] +│ SortPreservingMergeExec: [number@0 ASC NULLS LAST] +│ [Stage 2] => NetworkCoalesceExec: output_partitions=32, input_tasks=2 +└────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p15] t1:[p0..p15] + │ SortExec: expr=[number@0 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[number@0 as number], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=16, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p31] t1:[p0..p31] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([number@0], 32), input_partitions=16 + │ AggregateExec: mode=Partial, gby=[number@0 as number], aggr=[] + │ RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 + │ CooperativeExec + │ NumbersExec: t0:[0-6), t1:[6-11) + └────────────────────────────────────────────────── +``` + +The distribution rule kicks in, and the plan gets distributed. + +Note that the parallelism in the plan has an upper threshold, so for example, if we query 100 numbers: + +```bash +cargo run \ + --features integration \ + --example custom_execution_plan \ + "SELECT DISTINCT number FROM numbers(0, 100) ORDER BY number" \ + --show-distributed-plan +``` + +``` +┌───── DistributedExec ── Tasks: t0:[p0] +│ SortPreservingMergeExec: [number@0 ASC NULLS LAST] +│ [Stage 2] => NetworkCoalesceExec: output_partitions=48, input_tasks=3 +└────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p15] t1:[p0..p15] t2:[p0..p15] + │ SortExec: expr=[number@0 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[number@0 as number], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=16, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p47] t1:[p0..p47] t2:[p0..p47] t3:[p0..p47] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([number@0], 48), input_partitions=16 + │ AggregateExec: mode=Partial, gby=[number@0 as number], aggr=[] + │ RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 + │ CooperativeExec + │ NumbersExec: t0:[0-25), t1:[25-50), t2:[50-75), t3:[75-100) + └────────────────────────────────────────────────── +``` + +We do not get 100/10 = 10 distributed tasks, we just get 4. This is because the example is configured by default to +simulate a 4-worker cluster. If we increase the worker count, we get a highly distributed plan out with 10 tasks: + +```bash +cargo run \ + --features integration \ + --example custom_execution_plan \ + "SELECT DISTINCT number FROM numbers(0, 100) ORDER BY number" \ + --workers 10 \ + --show-distributed-plan +``` + +``` +┌───── DistributedExec ── Tasks: t0:[p0] +│ SortPreservingMergeExec: [number@0 ASC NULLS LAST] +│ [Stage 2] => NetworkCoalesceExec: output_partitions=112, input_tasks=7 +└────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p15] t1:[p0..p15] t2:[p0..p15] t3:[p0..p15] t4:[p0..p15] t5:[p0..p15] t6:[p0..p15] + │ SortExec: expr=[number@0 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[number@0 as number], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=16, input_tasks=10 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p111] t1:[p0..p111] t2:[p0..p111] t3:[p0..p111] t4:[p0..p111] t5:[p0..p111] t6:[p0..p111] t7:[p0..p111] t8:[p0..p111] t9:[p0..p111] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([number@0], 112), input_partitions=16 + │ AggregateExec: mode=Partial, gby=[number@0 as number], aggr=[] + │ RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 + │ CooperativeExec + │ NumbersExec: t0:[0-10), t1:[10-20), t2:[20-30), t3:[30-40), t4:[40-50), t5:[50-60), t6:[60-70), t7:[70-80), t8:[80-90), t9:[90-100) + └────────────────────────────────────────────────── +``` + diff --git a/examples/custom_execution_plan.rs b/examples/custom_execution_plan.rs new file mode 100644 index 00000000..10c16d28 --- /dev/null +++ b/examples/custom_execution_plan.rs @@ -0,0 +1,408 @@ +//! This example demonstrates how to create a custom execution plan that works with +//! Distributed DataFusion. It implements a `numbers(start, end)` table function that +//! generates a sequence of numbers and can be distributed across multiple workers. +//! +//! This example includes: +//! - Custom TableFunction for accepting the `numbers(start, end)` in SQL +//! - Custom TableProvider for mapping the table function to an execution plan +//! - Custom ExecutionPlan for returning the requested number range +//! - Custom PhysicalExtensionCodec for serialization across the network +//! - Custom TaskEstimator to control parallelism +//! +//! Run this example with: +//! ```bash +//! cargo run --features integration --example custom_execution_plan "SELECT DISTINCT number FROM numbers(0, 10) ORDER BY number" --show-distributed-plan +//! cargo run --features integration --example custom_execution_plan "SELECT DISTINCT number FROM numbers(0, 11) ORDER BY number" --show-distributed-plan +//! cargo run --features integration --example custom_execution_plan "SELECT DISTINCT number FROM numbers(0, 100) ORDER BY number" --show-distributed-plan +//! cargo run --features integration --example custom_execution_plan "SELECT DISTINCT number FROM numbers(0, 100) ORDER BY number" --workers 10 --show-distributed-plan +//! ``` + +use arrow::array::{ArrayRef, Int64Array, RecordBatch}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::record_batch::RecordBatchOptions; +use arrow::util::pretty::pretty_format_batches; +use async_trait::async_trait; +use datafusion::catalog::{Session, TableFunctionImpl}; +use datafusion::common::{ + DataFusionError, Result, ScalarValue, exec_err, extensions_options, internal_err, plan_err, +}; +use datafusion::config::ConfigExtension; +use datafusion::datasource::{TableProvider, TableType}; +use datafusion::execution::{SendableRecordBatchStream, SessionStateBuilder, TaskContext}; +use datafusion::logical_expr::Expr; +use datafusion::physical_expr::EquivalenceProperties; +use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; +use datafusion::prelude::{SessionConfig, SessionContext}; +use datafusion_distributed::test_utils::in_memory_channel_resolver::{ + InMemoryChannelResolver, InMemoryWorkerResolver, +}; +use datafusion_distributed::{ + DistributedExt, DistributedPhysicalOptimizerRule, DistributedTaskContext, TaskEstimation, + TaskEstimator, WorkerQueryContext, display_plan_ascii, +}; +use datafusion_proto::physical_plan::PhysicalExtensionCodec; +use datafusion_proto::protobuf; +use datafusion_proto::protobuf::proto_error; +use futures::{TryStreamExt, stream}; +use prost::Message; +use std::any::Any; +use std::fmt::{self, Formatter}; +use std::ops::Range; +use std::sync::Arc; +use structopt::StructOpt; + +/// Table function that generates a sequence of numbers from start to end. +/// Can be called in SQL as: SELECT * FROM numbers(start, end) +#[derive(Debug)] +struct NumbersTableFunction; + +impl TableFunctionImpl for NumbersTableFunction { + fn call(&self, exprs: &[Expr]) -> Result> { + if exprs.len() != 2 { + return plan_err!( + "numbers() requires exactly 2 arguments (start, end), got {}", + exprs.len() + ); + } + fn get_number(expr: &Expr) -> Result { + match &expr { + Expr::Literal(ScalarValue::Int64(Some(v)), _) => Ok(*v), + Expr::Literal(ScalarValue::Int32(Some(v)), _) => Ok(*v as i64), + v => plan_err!("numbers() arguments must be integer literals, got {v:?}"), + } + } + Ok(Arc::new(NumbersTableProvider { + start: get_number(&exprs[0])?, + end: get_number(&exprs[1])?, + })) + } +} + +fn numbers_schema() -> SchemaRef { + Arc::new(Schema::new(vec![Field::new( + "number", + DataType::Int64, + false, + )])) +} + +/// TableProvider that generates a sequence of numbers from start to end. +#[derive(Debug)] +struct NumbersTableProvider { + start: i64, + end: i64, +} + +#[async_trait] +impl TableProvider for NumbersTableProvider { + fn as_any(&self) -> &dyn Any { + self + } + + fn schema(&self) -> SchemaRef { + numbers_schema() + } + + fn table_type(&self) -> TableType { + TableType::Base + } + + async fn scan( + &self, + _state: &dyn Session, + projection: Option<&Vec>, + _filters: &[Expr], + _limit: Option, + ) -> Result> { + let schema = match projection { + Some(indices) => Arc::new(self.schema().project(indices)?), + None => self.schema(), + }; + + #[allow(clippy::single_range_in_vec_init)] + Ok(Arc::new(NumbersExec::new([self.start..self.end], schema))) + } +} + +/// Custom execution plan that generates numbers from start to end. +/// When distributed, each task generates a subset of numbers based on its task_index. +#[derive(Debug, Clone)] +struct NumbersExec { + ranges_per_task: Vec>, + plan_properties: PlanProperties, +} + +impl NumbersExec { + fn new(ranges_per_task: impl IntoIterator>, schema: SchemaRef) -> Self { + let plan_properties = PlanProperties::new( + EquivalenceProperties::new(schema.clone()), + datafusion::physical_expr::Partitioning::UnknownPartitioning(1), + EmissionType::Incremental, + Boundedness::Bounded, + ); + Self { + ranges_per_task: ranges_per_task.into_iter().collect(), + plan_properties, + } + } +} + +impl DisplayAs for NumbersExec { + fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> fmt::Result { + write!(f, "NumbersExec: ")?; + for (task_i, range) in self.ranges_per_task.iter().enumerate() { + write!(f, "t{task_i}:[{}-{})", range.start, range.end)?; + if task_i < self.ranges_per_task.len() - 1 { + write!(f, ", ")?; + } + } + Ok(()) + } +} + +impl ExecutionPlan for NumbersExec { + fn name(&self) -> &str { + "NumbersExec" + } + + fn as_any(&self) -> &dyn Any { + self + } + + fn properties(&self) -> &PlanProperties { + &self.plan_properties + } + + fn children(&self) -> Vec<&Arc> { + vec![] + } + + fn with_new_children( + self: Arc, + _: Vec>, + ) -> Result> { + Ok(self) + } + + fn execute( + &self, + _partition: usize, + context: Arc, + ) -> Result { + // Get the distributed task context to determine which subset of numbers + // this task should generate + let dist_ctx = DistributedTaskContext::from_ctx(&context); + + let Some(range) = self.ranges_per_task.get(dist_ctx.task_index) else { + return exec_err!("Task index out of range"); + }; + + // Calculate which numbers this task should generate + let numbers: Vec = range.clone().collect(); + let row_count = numbers.len(); + + // Create batch matching the schema (may be empty for COUNT queries) + let batch = if self.schema().fields().is_empty() { + // For COUNT queries, return batch with correct row count but no columns + let mut options = RecordBatchOptions::new(); + options.row_count = Some(row_count); + RecordBatch::try_new_with_options(self.schema(), vec![], &options)? + } else { + let array: ArrayRef = Arc::new(Int64Array::from(numbers)); + RecordBatch::try_new(self.schema(), vec![array])? + }; + + Ok(Box::pin(RecordBatchStreamAdapter::new( + self.schema(), + stream::once(async { Ok(batch) }), + ))) + } +} + +/// Custom codec for serializing/deserializing NumbersExec across the network. As the NumbersExec +/// plan will be sent over the wire during distributed queries, both the SessionContext that +/// initiates the query and each Worker need to know how to (de)serialize it. +#[derive(Debug)] +struct NumbersExecCodec; + +#[derive(Clone, PartialEq, ::prost::Message)] +struct NumbersExecProto { + #[prost(message, optional, tag = "1")] + schema: Option, + #[prost(repeated, message, tag = "2")] + ranges: Vec, +} + +#[derive(Clone, PartialEq, ::prost::Message)] +struct RangeProto { + #[prost(int64, tag = "1")] + start: i64, + #[prost(int64, tag = "2")] + end: i64, +} + +impl PhysicalExtensionCodec for NumbersExecCodec { + fn try_decode( + &self, + buf: &[u8], + inputs: &[Arc], + _ctx: &TaskContext, + ) -> Result> { + if !inputs.is_empty() { + return internal_err!("NumbersExec should have no children, got {}", inputs.len()); + } + + let proto = NumbersExecProto::decode(buf) + .map_err(|e| proto_error(format!("Failed to decode NumbersExec: {e}")))?; + + let schema: Schema = proto + .schema + .as_ref() + .map(|s| s.try_into()) + .ok_or(proto_error("NetworkShuffleExec is missing schema"))??; + + Ok(Arc::new(NumbersExec::new( + proto.ranges.iter().map(|v| v.start..v.end), + Arc::new(schema), + ))) + } + + fn try_encode(&self, node: Arc, buf: &mut Vec) -> Result<()> { + let Some(exec) = node.as_any().downcast_ref::() else { + return internal_err!("Expected plan to be NumbersExec, but was {}", node.name()); + }; + + let proto = NumbersExecProto { + schema: Some(node.schema().try_into()?), + ranges: exec + .ranges_per_task + .iter() + .map(|v| RangeProto { + start: v.start, + end: v.end, + }) + .collect(), + }; + + proto + .encode(buf) + .map_err(|e| proto_error(format!("Failed to encode NumbersExec: {e}"))) + } +} + +extensions_options! { + /// Custom ConfigExtension for configuring NumbersExec distributed task estimation behavior + /// at runtime with SET statements. + struct NumbersConfig { + /// how many numbers each task will produce + numbers_per_task: usize, default = 10 + } +} + +impl ConfigExtension for NumbersConfig { + const PREFIX: &'static str = "numbers"; +} + +/// Custom TaskEstimator that tells the planner how to distribute NumbersExec. +#[derive(Debug)] +struct NumbersTaskEstimator; + +impl TaskEstimator for NumbersTaskEstimator { + fn task_estimation( + &self, + plan: &Arc, + cfg: &datafusion::config::ConfigOptions, + ) -> Option { + let plan = plan.as_any().downcast_ref::()?; + let cfg: &NumbersConfig = cfg.extensions.get()?; + let task_count = (plan.ranges_per_task[0].end - plan.ranges_per_task[0].start) as f64 + / cfg.numbers_per_task as f64; + + Some(TaskEstimation::desired(task_count.ceil() as usize)) + } + + fn scale_up_leaf_node( + &self, + plan: &Arc, + task_count: usize, + _cfg: &datafusion::config::ConfigOptions, + ) -> Option> { + let plan = plan.as_any().downcast_ref::()?; + let range = &plan.ranges_per_task[0]; + let chunk_size = ((range.end - range.start) as f64 / task_count as f64).ceil() as i64; + + let ranges_per_task = (0..task_count).map(|i| { + let start = range.start + (i as i64 * chunk_size); + let end = (start + chunk_size).min(range.end); + start..end + }); + + Some(Arc::new(NumbersExec::new(ranges_per_task, plan.schema()))) + } +} + +#[derive(StructOpt)] +#[structopt( + name = "custom_execution_plan", + about = "Example demonstrating custom execution plans with Distributed DataFusion" +)] +struct Args { + /// The SQL query to run. + #[structopt()] + query: String, + + /// Number of distributed workers to simulate. + #[structopt(long, default_value = "4")] + workers: usize, + + /// Whether the distributed plan should be rendered instead of executing the query. + #[structopt(long)] + show_distributed_plan: bool, +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let args = Args::from_args(); + + let worker_resolver = InMemoryWorkerResolver::new(args.workers); + let channel_resolver = + InMemoryChannelResolver::from_session_builder(|ctx: WorkerQueryContext| async move { + Ok(ctx + .builder + .with_distributed_user_codec(NumbersExecCodec) + .build()) + }); + + let config = SessionConfig::new().with_option_extension(NumbersConfig::default()); + + let state = SessionStateBuilder::new() + .with_default_features() + .with_config(config) + .with_distributed_worker_resolver(worker_resolver) + .with_distributed_channel_resolver(channel_resolver) + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + .with_distributed_user_codec(NumbersExecCodec) + .with_distributed_task_estimator(NumbersTaskEstimator) + .build(); + + let ctx = SessionContext::from(state); + ctx.register_udtf("numbers", Arc::new(NumbersTableFunction)); + + let mut df = None; + for query in args.query.split(';') { + df = Some(ctx.sql(query).await?); + } + let df = df.unwrap(); + if args.show_distributed_plan { + let plan = df.create_physical_plan().await?; + println!("{}", display_plan_ascii(plan.as_ref(), false)); + } else { + let stream = df.execute_stream().await?; + let batches = stream.try_collect::>().await?; + let formatted = pretty_format_batches(&batches)?; + println!("{formatted}"); + } + Ok(()) +} From 086b89f096ad0fc59b679be34ad1e68574e681d7 Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Tue, 6 Jan 2026 20:32:11 +0100 Subject: [PATCH 14/27] Add ballista to benchmarks (#278) --- Cargo.lock | 2102 ++++++++++++++---- benchmarks/Cargo.toml | 17 + benchmarks/cdk/bin/ballista-bench.ts | 97 + benchmarks/cdk/bin/ballista_executor.rs | 36 + benchmarks/cdk/bin/ballista_http.rs | 124 ++ benchmarks/cdk/bin/ballista_scheduler.rs | 62 + benchmarks/cdk/bin/cdk.ts | 14 +- benchmarks/cdk/lib/ballista.ts | 212 ++ benchmarks/cdk/lib/cdk-stack.ts | 408 ++-- benchmarks/cdk/lib/datafusion-distributed.ts | 73 + benchmarks/cdk/lib/trino.ts | 119 +- benchmarks/cdk/package.json | 1 + 12 files changed, 2641 insertions(+), 624 deletions(-) create mode 100644 benchmarks/cdk/bin/ballista-bench.ts create mode 100644 benchmarks/cdk/bin/ballista_executor.rs create mode 100644 benchmarks/cdk/bin/ballista_http.rs create mode 100644 benchmarks/cdk/bin/ballista_scheduler.rs create mode 100644 benchmarks/cdk/lib/ballista.ts create mode 100644 benchmarks/cdk/lib/datafusion-distributed.ts diff --git a/Cargo.lock b/Cargo.lock index c6e3043f..64242eed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -158,8 +158,8 @@ dependencies = [ "serde_bytes", "serde_json", "snap", - "strum", - "strum_macros", + "strum 0.27.2", + "strum_macros 0.27.2", "thiserror", "uuid", "xz2", @@ -196,25 +196,60 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "arrow" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc" +dependencies = [ + "arrow-arith 56.2.0", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-csv 56.2.0", + "arrow-data 56.2.0", + "arrow-ipc 56.2.0", + "arrow-json 56.2.0", + "arrow-ord 56.2.0", + "arrow-row 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "arrow-string 56.2.0", +] + [[package]] name = "arrow" version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb372a7cbcac02a35d3fb7b3fc1f969ec078e871f9bb899bf00a2e1809bec8a3" dependencies = [ - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-csv", - "arrow-data", - "arrow-ipc", - "arrow-json", - "arrow-ord", - "arrow-row", - "arrow-schema", - "arrow-select", - "arrow-string", + "arrow-arith 57.1.0", + "arrow-array 57.1.0", + "arrow-buffer 57.1.0", + "arrow-cast 57.1.0", + "arrow-csv 57.1.0", + "arrow-data 57.1.0", + "arrow-ipc 57.1.0", + "arrow-json 57.1.0", + "arrow-ord 57.1.0", + "arrow-row 57.1.0", + "arrow-schema 57.1.0", + "arrow-select 57.1.0", + "arrow-string 57.1.0", +] + +[[package]] +name = "arrow-arith" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "chrono", + "num", ] [[package]] @@ -223,14 +258,31 @@ version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f377dcd19e440174596d83deb49cd724886d91060c07fec4f67014ef9d54049" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 57.1.0", + "arrow-buffer 57.1.0", + "arrow-data 57.1.0", + "arrow-schema 57.1.0", "chrono", "num-traits", ] +[[package]] +name = "arrow-array" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" +dependencies = [ + "ahash", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.16.1", + "num", +] + [[package]] name = "arrow-array" version = "57.1.0" @@ -238,9 +290,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eaff85a44e9fa914660fb0d0bb00b79c4a3d888b5334adb3ea4330c84f002" dependencies = [ "ahash", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 57.1.0", + "arrow-data 57.1.0", + "arrow-schema 57.1.0", "chrono", "chrono-tz", "half", @@ -250,6 +302,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-buffer" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" +dependencies = [ + "bytes", + "half", + "num", +] + [[package]] name = "arrow-buffer" version = "57.1.0" @@ -262,18 +325,39 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-cast" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num", + "ryu", +] + [[package]] name = "arrow-cast" version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3d131abb183f80c450d4591dc784f8d7750c50c6e2bc3fcaad148afc8361271" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-ord", - "arrow-schema", - "arrow-select", + "arrow-array 57.1.0", + "arrow-buffer 57.1.0", + "arrow-data 57.1.0", + "arrow-ord 57.1.0", + "arrow-schema 57.1.0", + "arrow-select 57.1.0", "atoi", "base64", "chrono", @@ -284,81 +368,173 @@ dependencies = [ "ryu", ] +[[package]] +name = "arrow-csv" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa9bf02705b5cf762b6f764c65f04ae9082c7cfc4e96e0c33548ee3f67012eb" +dependencies = [ + "arrow-array 56.2.0", + "arrow-cast 56.2.0", + "arrow-schema 56.2.0", + "chrono", + "csv", + "csv-core", + "regex", +] + [[package]] name = "arrow-csv" version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2275877a0e5e7e7c76954669366c2aa1a829e340ab1f612e647507860906fb6b" dependencies = [ - "arrow-array", - "arrow-cast", - "arrow-schema", + "arrow-array 57.1.0", + "arrow-cast 57.1.0", + "arrow-schema 57.1.0", "chrono", "csv", "csv-core", "regex", ] +[[package]] +name = "arrow-data" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" +dependencies = [ + "arrow-buffer 56.2.0", + "arrow-schema 56.2.0", + "half", + "num", +] + [[package]] name = "arrow-data" version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05738f3d42cb922b9096f7786f606fcb8669260c2640df8490533bb2fa38c9d3" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 57.1.0", + "arrow-schema 57.1.0", "half", "num-integer", "num-traits", ] +[[package]] +name = "arrow-flight" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c8b0ba0784d56bc6266b79f5de7a24b47024e7b3a0045d2ad4df3d9b686099f" +dependencies = [ + "arrow-arith 56.2.0", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-data 56.2.0", + "arrow-ipc 56.2.0", + "arrow-ord 56.2.0", + "arrow-row 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "arrow-string 56.2.0", + "base64", + "bytes", + "futures", + "once_cell", + "paste", + "prost 0.13.5", + "prost-types 0.13.5", + "tonic 0.13.1", +] + [[package]] name = "arrow-flight" version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b5f57c3d39d1b1b7c1376a772ea86a131e7da310aed54ebea9363124bb885e3" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-ipc", - "arrow-schema", + "arrow-array 57.1.0", + "arrow-buffer 57.1.0", + "arrow-cast 57.1.0", + "arrow-ipc 57.1.0", + "arrow-schema 57.1.0", "base64", "bytes", "futures", - "prost", - "prost-types", - "tonic", + "prost 0.14.1", + "prost-types 0.14.1", + "tonic 0.14.2", "tonic-prost", ] +[[package]] +name = "arrow-ipc" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d3594dcddccc7f20fd069bc8e9828ce37220372680ff638c5e00dea427d88f5" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "flatbuffers", + "lz4_flex 0.11.5", + "zstd", +] + [[package]] name = "arrow-ipc" version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d09446e8076c4b3f235603d9ea7c5494e73d441b01cd61fb33d7254c11964b3" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 57.1.0", + "arrow-buffer 57.1.0", + "arrow-data 57.1.0", + "arrow-schema 57.1.0", + "arrow-select 57.1.0", "flatbuffers", - "lz4_flex", + "lz4_flex 0.12.0", "zstd", ] +[[package]] +name = "arrow-json" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88cf36502b64a127dc659e3b305f1d993a544eab0d48cce704424e62074dc04b" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "chrono", + "half", + "indexmap", + "lexical-core", + "memchr", + "num", + "serde", + "serde_json", + "simdutf8", +] + [[package]] name = "arrow-json" version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "371ffd66fa77f71d7628c63f209c9ca5341081051aa32f9c8020feb0def787c0" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-array 57.1.0", + "arrow-buffer 57.1.0", + "arrow-cast 57.1.0", + "arrow-data 57.1.0", + "arrow-schema 57.1.0", "chrono", "half", "indexmap", @@ -372,17 +548,43 @@ dependencies = [ "simdutf8", ] +[[package]] +name = "arrow-ord" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", +] + [[package]] name = "arrow-ord" version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbc94fc7adec5d1ba9e8cd1b1e8d6f72423b33fe978bf1f46d970fafab787521" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 57.1.0", + "arrow-buffer 57.1.0", + "arrow-data 57.1.0", + "arrow-schema 57.1.0", + "arrow-select 57.1.0", +] + +[[package]] +name = "arrow-row" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "half", ] [[package]] @@ -391,13 +593,23 @@ version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "169676f317157dc079cc5def6354d16db63d8861d61046d2f3883268ced6f99f" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 57.1.0", + "arrow-buffer 57.1.0", + "arrow-data 57.1.0", + "arrow-schema 57.1.0", "half", ] +[[package]] +name = "arrow-schema" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "arrow-schema" version = "57.1.0" @@ -408,6 +620,20 @@ dependencies = [ "serde_json", ] +[[package]] +name = "arrow-select" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" +dependencies = [ + "ahash", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "num", +] + [[package]] name = "arrow-select" version = "57.1.0" @@ -415,24 +641,41 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae980d021879ea119dd6e2a13912d81e64abed372d53163e804dfe84639d8010" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 57.1.0", + "arrow-buffer 57.1.0", + "arrow-data 57.1.0", + "arrow-schema 57.1.0", "num-traits", ] +[[package]] +name = "arrow-string" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "memchr", + "num", + "regex", + "regex-syntax", +] + [[package]] name = "arrow-string" version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf35e8ef49dcf0c5f6d175edee6b8af7b45611805333129c541a8b89a0fc0534" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 57.1.0", + "arrow-buffer 57.1.0", + "arrow-data 57.1.0", + "arrow-schema 57.1.0", + "arrow-select 57.1.0", "memchr", "num-traits", "regex", @@ -951,10 +1194,13 @@ checksum = "5b098575ebe77cb6d14fc7f32749631a6e44edbef6b796f89b020e99ba20d425" dependencies = [ "axum-core 0.5.5", "bytes", + "form_urlencoded", "futures-util", "http 1.3.1", "http-body 1.0.1", "http-body-util", + "hyper 1.8.1", + "hyper-util", "itoa", "matchit 0.8.4", "memchr", @@ -962,10 +1208,15 @@ dependencies = [ "percent-encoding", "pin-project-lite", "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", "sync_wrapper", + "tokio", "tower", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -1005,6 +1256,119 @@ dependencies = [ "sync_wrapper", "tower-layer", "tower-service", + "tracing", +] + +[[package]] +name = "ballista" +version = "50.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f2a3159e4138cf7de0ba8b7267aa97eb0c01d960920b1747dba71e6986ef421" +dependencies = [ + "async-trait", + "ballista-core", + "ballista-executor", + "ballista-scheduler", + "datafusion 50.3.0", + "log", + "tokio", + "url", +] + +[[package]] +name = "ballista-core" +version = "50.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca9c994b0f51d52a219fbaf4ca1769f807795a998e9d3c50a03ced67693860e" +dependencies = [ + "arrow-flight 56.2.0", + "async-trait", + "aws-config", + "aws-credential-types", + "chrono", + "clap 4.5.53", + "datafusion 50.3.0", + "datafusion-proto 50.3.0", + "datafusion-proto-common 50.3.0", + "futures", + "itertools", + "log", + "md-5", + "object_store", + "parking_lot", + "prost 0.13.5", + "prost-types 0.13.5", + "rand 0.9.2", + "rustc_version", + "serde", + "tokio", + "tokio-stream", + "tonic 0.13.1", + "tonic-build", + "url", + "uuid", +] + +[[package]] +name = "ballista-executor" +version = "50.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad1bf32e64828067a003f201fc0068da4863867936a680af6ca3af7edda94d6e" +dependencies = [ + "arrow 56.2.0", + "arrow-flight 56.2.0", + "async-trait", + "ballista-core", + "clap 4.5.53", + "dashmap", + "datafusion 50.3.0", + "datafusion-proto 50.3.0", + "futures", + "libc", + "log", + "mimalloc", + "parking_lot", + "tempfile", + "tokio", + "tokio-stream", + "tokio-util", + "tonic 0.13.1", + "tracing", + "tracing-appender", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "ballista-scheduler" +version = "50.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54e618c0dbe63c48f69b417b26a2824a6be849bbf78fd73e7d98b567ce5907f3" +dependencies = [ + "arrow-flight 56.2.0", + "async-trait", + "axum 0.8.7", + "ballista-core", + "clap 4.5.53", + "dashmap", + "datafusion 50.3.0", + "datafusion-proto 50.3.0", + "futures", + "http 1.3.1", + "log", + "object_store", + "parking_lot", + "prost 0.13.5", + "prost-types 0.13.5", + "rand 0.9.2", + "serde", + "tokio", + "tokio-stream", + "tonic 0.13.1", + "tracing", + "tracing-appender", + "tracing-subscriber", + "uuid", ] [[package]] @@ -1321,11 +1685,12 @@ checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "comfy-table" -version = "7.2.1" +version = "7.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b03b7db8e0b4b2fdad6c551e634134e99ec000e5c8c3b6856c65e8bbaded7a3b" +checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56" dependencies = [ - "unicode-segmentation", + "strum 0.26.3", + "strum_macros 0.26.4", "unicode-width 0.2.2", ] @@ -1536,55 +1901,110 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "datafusion" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af15bb3c6ffa33011ef579f6b0bcbe7c26584688bd6c994f548e44df67f011a" +dependencies = [ + "arrow 56.2.0", + "arrow-ipc 56.2.0", + "arrow-schema 56.2.0", + "async-trait", + "bytes", + "bzip2 0.6.1", + "chrono", + "datafusion-catalog 50.3.0", + "datafusion-catalog-listing 50.3.0", + "datafusion-common 50.3.0", + "datafusion-common-runtime 50.3.0", + "datafusion-datasource 50.3.0", + "datafusion-datasource-csv 50.3.0", + "datafusion-datasource-json 50.3.0", + "datafusion-datasource-parquet 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-expr-common 50.3.0", + "datafusion-functions 50.3.0", + "datafusion-functions-aggregate 50.3.0", + "datafusion-functions-nested 50.3.0", + "datafusion-functions-table 50.3.0", + "datafusion-functions-window 50.3.0", + "datafusion-optimizer 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-adapter 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "datafusion-physical-optimizer 50.3.0", + "datafusion-physical-plan 50.3.0", + "datafusion-session 50.3.0", + "datafusion-sql 50.3.0", + "flate2", + "futures", + "itertools", + "log", + "object_store", + "parking_lot", + "parquet 56.2.0", + "rand 0.9.2", + "regex", + "sqlparser 0.58.0", + "tempfile", + "tokio", + "url", + "uuid", + "xz2", + "zstd", +] + [[package]] name = "datafusion" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ba7cb113e9c0bedf9e9765926031e132fa05a1b09ba6e93a6d1a4d7044457b8" dependencies = [ - "arrow", - "arrow-schema", + "arrow 57.1.0", + "arrow-schema 57.1.0", "async-trait", "bytes", "bzip2 0.6.1", "chrono", - "datafusion-catalog", - "datafusion-catalog-listing", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", + "datafusion-catalog 51.0.0", + "datafusion-catalog-listing 51.0.0", + "datafusion-common 51.0.0", + "datafusion-common-runtime 51.0.0", + "datafusion-datasource 51.0.0", "datafusion-datasource-arrow", "datafusion-datasource-avro", - "datafusion-datasource-csv", - "datafusion-datasource-json", - "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", - "datafusion-functions-table", - "datafusion-functions-window", - "datafusion-optimizer", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-session", - "datafusion-sql", + "datafusion-datasource-csv 51.0.0", + "datafusion-datasource-json 51.0.0", + "datafusion-datasource-parquet 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-expr-common 51.0.0", + "datafusion-functions 51.0.0", + "datafusion-functions-aggregate 51.0.0", + "datafusion-functions-nested 51.0.0", + "datafusion-functions-table 51.0.0", + "datafusion-functions-window 51.0.0", + "datafusion-optimizer 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-adapter 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-optimizer 51.0.0", + "datafusion-physical-plan 51.0.0", + "datafusion-session 51.0.0", + "datafusion-sql 51.0.0", "flate2", "futures", "itertools", "log", "object_store", "parking_lot", - "parquet", + "parquet 57.1.0", "rand 0.9.2", "regex", "rstest", - "sqlparser", + "sqlparser 0.59.0", "tempfile", "tokio", "url", @@ -1595,21 +2015,22 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "51.0.0" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a3a799f914a59b1ea343906a0486f17061f39509af74e874a866428951130d" +checksum = "187622262ad8f7d16d3be9202b4c1e0116f1c9aa387e5074245538b755261621" dependencies = [ - "arrow", + "arrow 56.2.0", "async-trait", "dashmap", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 50.3.0", + "datafusion-common-runtime 50.3.0", + "datafusion-datasource 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-plan 50.3.0", + "datafusion-session 50.3.0", + "datafusion-sql 50.3.0", "futures", "itertools", "log", @@ -1619,57 +2040,130 @@ dependencies = [ ] [[package]] -name = "datafusion-catalog-listing" +name = "datafusion-catalog" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db1b113c80d7a0febcd901476a57aef378e717c54517a163ed51417d87621b0" +checksum = "66a3a799f914a59b1ea343906a0486f17061f39509af74e874a866428951130d" dependencies = [ - "arrow", + "arrow 57.1.0", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", + "dashmap", + "datafusion-common 51.0.0", + "datafusion-common-runtime 51.0.0", + "datafusion-datasource 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-plan 51.0.0", + "datafusion-session 51.0.0", "futures", "itertools", "log", "object_store", + "parking_lot", "tokio", ] [[package]] -name = "datafusion-cli" -version = "51.0.0" +name = "datafusion-catalog-listing" +version = "50.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fab982df44f818a749cb5200504ccb919f4608cb9808daf8b3fb98aa7955fd1e" +checksum = "9657314f0a32efd0382b9a46fdeb2d233273ece64baa68a7c45f5a192daf0f83" dependencies = [ - "arrow", + "arrow 56.2.0", "async-trait", - "aws-config", - "aws-credential-types", - "chrono", - "clap 4.5.53", - "datafusion", - "datafusion-common", - "dirs", - "env_logger", + "datafusion-catalog 50.3.0", + "datafusion-common 50.3.0", + "datafusion-datasource 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "datafusion-physical-plan 50.3.0", + "datafusion-session 50.3.0", "futures", "log", - "mimalloc", "object_store", - "parking_lot", - "parquet", - "regex", + "tokio", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db1b113c80d7a0febcd901476a57aef378e717c54517a163ed51417d87621b0" +dependencies = [ + "arrow 57.1.0", + "async-trait", + "datafusion-catalog 51.0.0", + "datafusion-common 51.0.0", + "datafusion-datasource 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-adapter 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-plan 51.0.0", + "futures", + "itertools", + "log", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-cli" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fab982df44f818a749cb5200504ccb919f4608cb9808daf8b3fb98aa7955fd1e" +dependencies = [ + "arrow 57.1.0", + "async-trait", + "aws-config", + "aws-credential-types", + "chrono", + "clap 4.5.53", + "datafusion 51.0.0", + "datafusion-common 51.0.0", + "dirs", + "env_logger", + "futures", + "log", + "mimalloc", + "object_store", + "parking_lot", + "parquet 57.1.0", + "regex", "rustyline", "tokio", "url", ] +[[package]] +name = "datafusion-common" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a83760d9a13122d025fbdb1d5d5aaf93dd9ada5e90ea229add92aa30898b2d1" +dependencies = [ + "ahash", + "arrow 56.2.0", + "arrow-ipc 56.2.0", + "base64", + "chrono", + "half", + "hashbrown 0.14.5", + "indexmap", + "libc", + "log", + "object_store", + "parquet 56.2.0", + "paste", + "recursive", + "sqlparser 0.58.0", + "tokio", + "web-time", +] + [[package]] name = "datafusion-common" version = "51.0.0" @@ -1678,8 +2172,8 @@ checksum = "7c10f7659e96127d25e8366be7c8be4109595d6a2c3eac70421f380a7006a1b0" dependencies = [ "ahash", "apache-avro", - "arrow", - "arrow-ipc", + "arrow 57.1.0", + "arrow-ipc 57.1.0", "chrono", "half", "hashbrown 0.14.5", @@ -1688,14 +2182,25 @@ dependencies = [ "libc", "log", "object_store", - "parquet", + "parquet 57.1.0", "paste", "recursive", - "sqlparser", + "sqlparser 0.59.0", "tokio", "web-time", ] +[[package]] +name = "datafusion-common-runtime" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b6234a6c7173fe5db1c6c35c01a12b2aa0f803a3007feee53483218817f8b1e" +dependencies = [ + "futures", + "log", + "tokio", +] + [[package]] name = "datafusion-common-runtime" version = "51.0.0" @@ -1707,27 +2212,64 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-datasource" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7256c9cb27a78709dd42d0c80f0178494637209cac6e29d5c93edd09b6721b86" +dependencies = [ + "arrow 56.2.0", + "async-compression", + "async-trait", + "bytes", + "bzip2 0.6.1", + "chrono", + "datafusion-common 50.3.0", + "datafusion-common-runtime 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-adapter 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "datafusion-physical-plan 50.3.0", + "datafusion-session 50.3.0", + "flate2", + "futures", + "glob", + "itertools", + "log", + "object_store", + "parquet 56.2.0", + "rand 0.9.2", + "tempfile", + "tokio", + "tokio-util", + "url", + "xz2", + "zstd", +] + [[package]] name = "datafusion-datasource" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fde13794244bc7581cd82f6fff217068ed79cdc344cafe4ab2c3a1c3510b38d6" dependencies = [ - "arrow", + "arrow 57.1.0", "async-compression", "async-trait", "bytes", "bzip2 0.6.1", "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 51.0.0", + "datafusion-common-runtime 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-adapter 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-plan 51.0.0", + "datafusion-session 51.0.0", "flate2", "futures", "glob", @@ -1748,18 +2290,18 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "804fa9b4ecf3157982021770617200ef7c1b2979d57bec9044748314775a9aea" dependencies = [ - "arrow", - "arrow-ipc", + "arrow 57.1.0", + "arrow-ipc 57.1.0", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 51.0.0", + "datafusion-common-runtime 51.0.0", + "datafusion-datasource 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-plan 51.0.0", + "datafusion-session 51.0.0", "futures", "itertools", "object_store", @@ -1773,91 +2315,174 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388ed8be535f562cc655b9c3d22edbfb0f1a50a25c242647a98b6d92a75b55a1" dependencies = [ "apache-avro", - "arrow", + "arrow 57.1.0", "async-trait", "bytes", - "datafusion-common", - "datafusion-datasource", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 51.0.0", + "datafusion-datasource 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-plan 51.0.0", + "datafusion-session 51.0.0", "futures", "num-traits", "object_store", ] +[[package]] +name = "datafusion-datasource-csv" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64533a90f78e1684bfb113d200b540f18f268134622d7c96bbebc91354d04825" +dependencies = [ + "arrow 56.2.0", + "async-trait", + "bytes", + "datafusion-catalog 50.3.0", + "datafusion-common 50.3.0", + "datafusion-common-runtime 50.3.0", + "datafusion-datasource 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "datafusion-physical-plan 50.3.0", + "datafusion-session 50.3.0", + "futures", + "object_store", + "regex", + "tokio", +] + [[package]] name = "datafusion-datasource-csv" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61a1641a40b259bab38131c5e6f48fac0717bedb7dc93690e604142a849e0568" dependencies = [ - "arrow", + "arrow 57.1.0", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 51.0.0", + "datafusion-common-runtime 51.0.0", + "datafusion-datasource 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-plan 51.0.0", + "datafusion-session 51.0.0", "futures", "object_store", "regex", "tokio", ] +[[package]] +name = "datafusion-datasource-json" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d7ebeb12c77df0aacad26f21b0d033aeede423a64b2b352f53048a75bf1d6e6" +dependencies = [ + "arrow 56.2.0", + "async-trait", + "bytes", + "datafusion-catalog 50.3.0", + "datafusion-common 50.3.0", + "datafusion-common-runtime 50.3.0", + "datafusion-datasource 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "datafusion-physical-plan 50.3.0", + "datafusion-session 50.3.0", + "futures", + "object_store", + "serde_json", + "tokio", +] + [[package]] name = "datafusion-datasource-json" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adeacdb00c1d37271176f8fb6a1d8ce096baba16ea7a4b2671840c5c9c64fe85" dependencies = [ - "arrow", + "arrow 57.1.0", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 51.0.0", + "datafusion-common-runtime 51.0.0", + "datafusion-datasource 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-plan 51.0.0", + "datafusion-session 51.0.0", "futures", "object_store", "tokio", ] +[[package]] +name = "datafusion-datasource-parquet" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e783c4c7d7faa1199af2df4761c68530634521b176a8d1331ddbc5a5c75133" +dependencies = [ + "arrow 56.2.0", + "async-trait", + "bytes", + "datafusion-catalog 50.3.0", + "datafusion-common 50.3.0", + "datafusion-common-runtime 50.3.0", + "datafusion-datasource 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-functions-aggregate 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-adapter 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "datafusion-physical-optimizer 50.3.0", + "datafusion-physical-plan 50.3.0", + "datafusion-pruning 50.3.0", + "datafusion-session 50.3.0", + "futures", + "itertools", + "log", + "object_store", + "parking_lot", + "parquet 56.2.0", + "rand 0.9.2", + "tokio", +] + [[package]] name = "datafusion-datasource-parquet" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43d0b60ffd66f28bfb026565d62b0a6cbc416da09814766a3797bba7d85a3cd9" dependencies = [ - "arrow", + "arrow 57.1.0", "async-trait", "bytes", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", - "datafusion-session", + "datafusion-common 51.0.0", + "datafusion-common-runtime 51.0.0", + "datafusion-datasource 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-functions-aggregate-common 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-adapter 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-plan 51.0.0", + "datafusion-pruning 51.0.0", + "datafusion-session 51.0.0", "futures", "itertools", "log", "object_store", "parking_lot", - "parquet", + "parquet 57.1.0", "tokio", ] @@ -1865,15 +2490,15 @@ dependencies = [ name = "datafusion-distributed" version = "0.1.0" dependencies = [ - "arrow", - "arrow-flight", - "arrow-select", + "arrow 57.1.0", + "arrow-flight 57.1.0", + "arrow-select 57.1.0", "async-trait", "bytes", "chrono", "dashmap", - "datafusion", - "datafusion-proto", + "datafusion 51.0.0", + "datafusion-proto 51.0.0", "delegate", "futures", "http 1.3.1", @@ -1882,16 +2507,16 @@ dependencies = [ "itertools", "moka", "object_store", - "parquet", + "parquet 57.1.0", "pin-project", "pretty_assertions", - "prost", + "prost 0.14.1", "rand 0.8.5", "reqwest", "structopt", "tokio", "tokio-stream", - "tonic", + "tonic 0.14.2", "tower", "tpchgen", "tpchgen-arrow", @@ -1904,28 +2529,33 @@ dependencies = [ name = "datafusion-distributed-benchmarks" version = "0.1.0" dependencies = [ - "arrow-flight", + "arrow-flight 57.1.0", "async-trait", "aws-config", "aws-sdk-ec2", "axum 0.7.9", + "ballista", + "ballista-core", + "ballista-executor", + "ballista-scheduler", "chrono", + "clap 4.5.53", "dashmap", - "datafusion", + "datafusion 51.0.0", "datafusion-distributed", - "datafusion-proto", + "datafusion-proto 51.0.0", "env_logger", "futures", "log", "object_store", "openssl", - "parquet", - "prost", + "parquet 57.1.0", + "prost 0.14.1", "serde", "serde_json", "structopt", "tokio", - "tonic", + "tonic 0.14.2", "url", ] @@ -1933,10 +2563,10 @@ dependencies = [ name = "datafusion-distributed-cli" version = "0.1.0" dependencies = [ - "arrow-flight", + "arrow-flight 57.1.0", "async-trait", "clap 4.5.53", - "datafusion", + "datafusion 51.0.0", "datafusion-cli", "datafusion-distributed", "dirs", @@ -1944,59 +2574,120 @@ dependencies = [ "hyper-util", "tokio", "tokio-stream", - "tonic", + "tonic 0.14.2", "tower", "url", ] +[[package]] +name = "datafusion-doc" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ee6b1d9a80d13f9deb2291f45c07044b8e62fb540dbde2453a18be17a36429" + [[package]] name = "datafusion-doc" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b99e13947667b36ad713549237362afb054b2d8f8cc447751e23ec61202db07" +[[package]] +name = "datafusion-execution" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4cec0a57653bec7b933fb248d3ffa3fa3ab3bd33bd140dc917f714ac036f531" +dependencies = [ + "arrow 56.2.0", + "async-trait", + "dashmap", + "datafusion-common 50.3.0", + "datafusion-expr 50.3.0", + "futures", + "log", + "object_store", + "parking_lot", + "rand 0.9.2", + "tempfile", + "url", +] + [[package]] name = "datafusion-execution" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63695643190679037bc946ad46a263b62016931547bf119859c511f7ff2f5178" dependencies = [ - "arrow", + "arrow 57.1.0", "async-trait", "dashmap", - "datafusion-common", - "datafusion-expr", + "datafusion-common 51.0.0", + "datafusion-expr 51.0.0", "futures", "log", "object_store", "parking_lot", - "parquet", + "parquet 57.1.0", "rand 0.9.2", "tempfile", "url", ] +[[package]] +name = "datafusion-expr" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef76910bdca909722586389156d0aa4da4020e1631994d50fadd8ad4b1aa05fe" +dependencies = [ + "arrow 56.2.0", + "async-trait", + "chrono", + "datafusion-common 50.3.0", + "datafusion-doc 50.3.0", + "datafusion-expr-common 50.3.0", + "datafusion-functions-aggregate-common 50.3.0", + "datafusion-functions-window-common 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "indexmap", + "paste", + "recursive", + "serde_json", + "sqlparser 0.58.0", +] + [[package]] name = "datafusion-expr" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9a4787cbf5feb1ab351f789063398f67654a6df75c4d37d7f637dc96f951a91" dependencies = [ - "arrow", + "arrow 57.1.0", "async-trait", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", + "datafusion-common 51.0.0", + "datafusion-doc 51.0.0", + "datafusion-expr-common 51.0.0", + "datafusion-functions-aggregate-common 51.0.0", + "datafusion-functions-window-common 51.0.0", + "datafusion-physical-expr-common 51.0.0", "indexmap", "itertools", "paste", "recursive", "serde_json", - "sqlparser", + "sqlparser 0.59.0", +] + +[[package]] +name = "datafusion-expr-common" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d155ccbda29591ca71a1344dd6bed26c65a4438072b400df9db59447f590bb6" +dependencies = [ + "arrow 56.2.0", + "datafusion-common 50.3.0", + "indexmap", + "itertools", + "paste", ] [[package]] @@ -2005,31 +2696,60 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ce2fb1b8c15c9ac45b0863c30b268c69dc9ee7a1ee13ecf5d067738338173dc" dependencies = [ - "arrow", - "datafusion-common", + "arrow 57.1.0", + "datafusion-common 51.0.0", "indexmap", "itertools", "paste", ] +[[package]] +name = "datafusion-functions" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7de2782136bd6014670fd84fe3b0ca3b3e4106c96403c3ae05c0598577139977" +dependencies = [ + "arrow 56.2.0", + "arrow-buffer 56.2.0", + "base64", + "blake2", + "blake3", + "chrono", + "datafusion-common 50.3.0", + "datafusion-doc 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-expr-common 50.3.0", + "datafusion-macros 50.3.0", + "hex", + "itertools", + "log", + "md-5", + "rand 0.9.2", + "regex", + "sha2", + "unicode-segmentation", + "uuid", +] + [[package]] name = "datafusion-functions" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "794a9db7f7b96b3346fc007ff25e994f09b8f0511b4cf7dff651fadfe3ebb28f" dependencies = [ - "arrow", - "arrow-buffer", + "arrow 57.1.0", + "arrow-buffer 57.1.0", "base64", "blake2", "blake3", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-macros", + "datafusion-common 51.0.0", + "datafusion-doc 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-expr-common 51.0.0", + "datafusion-macros 51.0.0", "hex", "itertools", "log", @@ -2042,6 +2762,27 @@ dependencies = [ "uuid", ] +[[package]] +name = "datafusion-functions-aggregate" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07331fc13603a9da97b74fd8a273f4238222943dffdbbed1c4c6f862a30105bf" +dependencies = [ + "ahash", + "arrow 56.2.0", + "datafusion-common 50.3.0", + "datafusion-doc 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-functions-aggregate-common 50.3.0", + "datafusion-macros 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "half", + "log", + "paste", +] + [[package]] name = "datafusion-functions-aggregate" version = "51.0.0" @@ -2049,20 +2790,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c25210520a9dcf9c2b2cbbce31ebd4131ef5af7fc60ee92b266dc7d159cb305" dependencies = [ "ahash", - "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "arrow 57.1.0", + "datafusion-common 51.0.0", + "datafusion-doc 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-functions-aggregate-common 51.0.0", + "datafusion-macros 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-common 51.0.0", "half", "log", "paste", ] +[[package]] +name = "datafusion-functions-aggregate-common" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5951e572a8610b89968a09b5420515a121fbc305c0258651f318dc07c97ab17" +dependencies = [ + "ahash", + "arrow 56.2.0", + "datafusion-common 50.3.0", + "datafusion-expr-common 50.3.0", + "datafusion-physical-expr-common 50.3.0", +] + [[package]] name = "datafusion-functions-aggregate-common" version = "51.0.0" @@ -2070,10 +2824,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62f4a66f3b87300bb70f4124b55434d2ae3fe80455f3574701d0348da040b55d" dependencies = [ "ahash", - "arrow", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", + "arrow 57.1.0", + "datafusion-common 51.0.0", + "datafusion-expr-common 51.0.0", + "datafusion-physical-expr-common 51.0.0", +] + +[[package]] +name = "datafusion-functions-nested" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdacca9302c3d8fc03f3e94f338767e786a88a33f5ebad6ffc0e7b50364b9ea3" +dependencies = [ + "arrow 56.2.0", + "arrow-ord 56.2.0", + "datafusion-common 50.3.0", + "datafusion-doc 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-functions 50.3.0", + "datafusion-functions-aggregate 50.3.0", + "datafusion-functions-aggregate-common 50.3.0", + "datafusion-macros 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "itertools", + "log", + "paste", ] [[package]] @@ -2082,65 +2858,120 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae5c06eed03918dc7fe7a9f082a284050f0e9ecf95d72f57712d1496da03b8c4" dependencies = [ - "arrow", - "arrow-ord", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr-common", + "arrow 57.1.0", + "arrow-ord 57.1.0", + "datafusion-common 51.0.0", + "datafusion-doc 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-expr-common 51.0.0", + "datafusion-functions 51.0.0", + "datafusion-functions-aggregate 51.0.0", + "datafusion-functions-aggregate-common 51.0.0", + "datafusion-macros 51.0.0", + "datafusion-physical-expr-common 51.0.0", "itertools", "log", "paste", ] +[[package]] +name = "datafusion-functions-table" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c37ff8a99434fbbad604a7e0669717c58c7c4f14c472d45067c4b016621d981" +dependencies = [ + "arrow 56.2.0", + "async-trait", + "datafusion-catalog 50.3.0", + "datafusion-common 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-physical-plan 50.3.0", + "parking_lot", + "paste", +] + [[package]] name = "datafusion-functions-table" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db4fed1d71738fbe22e2712d71396db04c25de4111f1ec252b8f4c6d3b25d7f5" dependencies = [ - "arrow", + "arrow 57.1.0", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-plan", + "datafusion-catalog 51.0.0", + "datafusion-common 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-physical-plan 51.0.0", "parking_lot", "paste", ] +[[package]] +name = "datafusion-functions-window" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e2aea7c79c926cffabb13dc27309d4eaeb130f4a21c8ba91cdd241c813652b" +dependencies = [ + "arrow 56.2.0", + "datafusion-common 50.3.0", + "datafusion-doc 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-functions-window-common 50.3.0", + "datafusion-macros 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "log", + "paste", +] + [[package]] name = "datafusion-functions-window" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d92206aa5ae21892f1552b4d61758a862a70956e6fd7a95cb85db1de74bc6d1" dependencies = [ - "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "arrow 57.1.0", + "datafusion-common 51.0.0", + "datafusion-doc 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-functions-window-common 51.0.0", + "datafusion-macros 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-common 51.0.0", "log", "paste", ] +[[package]] +name = "datafusion-functions-window-common" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fead257ab5fd2ffc3b40fda64da307e20de0040fe43d49197241d9de82a487f" +dependencies = [ + "datafusion-common 50.3.0", + "datafusion-physical-expr-common 50.3.0", +] + [[package]] name = "datafusion-functions-window-common" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53ae9bcc39800820d53a22d758b3b8726ff84a5a3e24cecef04ef4e5fdf1c7cc" dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", + "datafusion-common 51.0.0", + "datafusion-physical-expr-common 51.0.0", +] + +[[package]] +name = "datafusion-macros" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec6f637bce95efac05cdfb9b6c19579ed4aa5f6b94d951cfa5bb054b7bb4f730" +dependencies = [ + "datafusion-expr 50.3.0", + "quote", + "syn 2.0.110", ] [[package]] @@ -2149,23 +2980,43 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1063ad4c9e094b3f798acee16d9a47bd7372d9699be2de21b05c3bd3f34ab848" dependencies = [ - "datafusion-doc", + "datafusion-doc 51.0.0", "quote", "syn 2.0.110", ] +[[package]] +name = "datafusion-optimizer" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6583ef666ae000a613a837e69e456681a9faa96347bf3877661e9e89e141d8a" +dependencies = [ + "arrow 56.2.0", + "chrono", + "datafusion-common 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-expr-common 50.3.0", + "datafusion-physical-expr 50.3.0", + "indexmap", + "itertools", + "log", + "recursive", + "regex", + "regex-syntax", +] + [[package]] name = "datafusion-optimizer" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f35f9ec5d08b87fd1893a30c2929f2559c2f9806ca072d8fefca5009dc0f06a" dependencies = [ - "arrow", + "arrow 57.1.0", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", + "datafusion-common 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-expr-common 51.0.0", + "datafusion-physical-expr 51.0.0", "indexmap", "itertools", "log", @@ -2174,6 +3025,29 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "datafusion-physical-expr" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8668103361a272cbbe3a61f72eca60c9b7c706e87cc3565bcf21e2b277b84f6" +dependencies = [ + "ahash", + "arrow 56.2.0", + "datafusion-common 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-expr-common 50.3.0", + "datafusion-functions-aggregate-common 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "half", + "hashbrown 0.14.5", + "indexmap", + "itertools", + "log", + "parking_lot", + "paste", + "petgraph 0.8.3", +] + [[package]] name = "datafusion-physical-expr" version = "51.0.0" @@ -2181,19 +3055,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c30cc8012e9eedcb48bbe112c6eff4ae5ed19cf3003cb0f505662e88b7014c5d" dependencies = [ "ahash", - "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", + "arrow 57.1.0", + "datafusion-common 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-expr-common 51.0.0", + "datafusion-functions-aggregate-common 51.0.0", + "datafusion-physical-expr-common 51.0.0", "half", "hashbrown 0.14.5", "indexmap", "itertools", "parking_lot", "paste", - "petgraph", + "petgraph 0.8.3", +] + +[[package]] +name = "datafusion-physical-expr-adapter" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "815acced725d30601b397e39958e0e55630e0a10d66ef7769c14ae6597298bb0" +dependencies = [ + "arrow 56.2.0", + "datafusion-common 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-functions 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "itertools", ] [[package]] @@ -2202,12 +3091,26 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f9ff2dbd476221b1f67337699eff432781c4e6e1713d2aefdaa517dfbf79768" dependencies = [ - "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "arrow 57.1.0", + "datafusion-common 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-functions 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "itertools", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6652fe7b5bf87e85ed175f571745305565da2c0b599d98e697bcbedc7baa47c3" +dependencies = [ + "ahash", + "arrow 56.2.0", + "datafusion-common 50.3.0", + "datafusion-expr-common 50.3.0", + "hashbrown 0.14.5", "itertools", ] @@ -2218,32 +3121,83 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90da43e1ec550b172f34c87ec68161986ced70fd05c8d2a2add66eef9c276f03" dependencies = [ "ahash", - "arrow", - "datafusion-common", - "datafusion-expr-common", + "arrow 57.1.0", + "datafusion-common 51.0.0", + "datafusion-expr-common 51.0.0", "hashbrown 0.14.5", "itertools", ] +[[package]] +name = "datafusion-physical-optimizer" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49b7d623eb6162a3332b564a0907ba00895c505d101b99af78345f1acf929b5c" +dependencies = [ + "arrow 56.2.0", + "datafusion-common 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-expr-common 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "datafusion-physical-plan 50.3.0", + "datafusion-pruning 50.3.0", + "itertools", + "log", + "recursive", +] + [[package]] name = "datafusion-physical-optimizer" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce9804f799acd7daef3be7aaffe77c0033768ed8fdbf5fb82fc4c5f2e6bc14e6" dependencies = [ - "arrow", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", + "arrow 57.1.0", + "datafusion-common 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-expr-common 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-plan 51.0.0", + "datafusion-pruning 51.0.0", "itertools", "recursive", ] +[[package]] +name = "datafusion-physical-plan" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2f7f778a1a838dec124efb96eae6144237d546945587557c9e6936b3414558c" +dependencies = [ + "ahash", + "arrow 56.2.0", + "arrow-ord 56.2.0", + "arrow-schema 56.2.0", + "async-trait", + "chrono", + "datafusion-common 50.3.0", + "datafusion-common-runtime 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-functions-aggregate-common 50.3.0", + "datafusion-functions-window-common 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "futures", + "half", + "hashbrown 0.14.5", + "indexmap", + "itertools", + "log", + "parking_lot", + "pin-project-lite", + "tokio", +] + [[package]] name = "datafusion-physical-plan" version = "51.0.0" @@ -2251,19 +3205,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0acf0ad6b6924c6b1aa7d213b181e012e2d3ec0a64ff5b10ee6282ab0f8532ac" dependencies = [ "ahash", - "arrow", - "arrow-ord", - "arrow-schema", + "arrow 57.1.0", + "arrow-ord 57.1.0", + "arrow-schema 57.1.0", "async-trait", "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 51.0.0", + "datafusion-common-runtime 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-functions-aggregate-common 51.0.0", + "datafusion-functions-window-common 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-common 51.0.0", "futures", "half", "hashbrown 0.14.5", @@ -2275,31 +3229,58 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-proto" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7df9f606892e6af45763d94d210634eec69b9bb6ced5353381682ff090028a3" +dependencies = [ + "arrow 56.2.0", + "chrono", + "datafusion 50.3.0", + "datafusion-common 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-proto-common 50.3.0", + "object_store", + "prost 0.13.5", +] + [[package]] name = "datafusion-proto" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d368093a98a17d1449b1083ac22ed16b7128e4c67789991869480d8c4a40ecb9" dependencies = [ - "arrow", + "arrow 57.1.0", "chrono", - "datafusion-catalog", - "datafusion-catalog-listing", - "datafusion-common", - "datafusion-datasource", + "datafusion-catalog 51.0.0", + "datafusion-catalog-listing 51.0.0", + "datafusion-common 51.0.0", + "datafusion-datasource 51.0.0", "datafusion-datasource-arrow", - "datafusion-datasource-csv", - "datafusion-datasource-json", - "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-table", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-proto-common", + "datafusion-datasource-csv 51.0.0", + "datafusion-datasource-json 51.0.0", + "datafusion-datasource-parquet 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-functions-table 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-plan 51.0.0", + "datafusion-proto-common 51.0.0", "object_store", - "prost", + "prost 0.14.1", +] + +[[package]] +name = "datafusion-proto-common" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4b14f288ca4ef77743d9672cafecf3adfffff0b9b04af9af79ecbeaaf736901" +dependencies = [ + "arrow 56.2.0", + "datafusion-common 50.3.0", + "prost 0.13.5", ] [[package]] @@ -2308,26 +3289,68 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b6aef3d5e5c1d2bc3114c4876730cb76a9bdc5a8df31ef1b6db48f0c1671895" dependencies = [ - "arrow", - "datafusion-common", - "prost", + "arrow 57.1.0", + "datafusion-common 51.0.0", + "prost 0.14.1", +] + +[[package]] +name = "datafusion-pruning" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1e59e2ca14fe3c30f141600b10ad8815e2856caa59ebbd0e3e07cd3d127a65" +dependencies = [ + "arrow 56.2.0", + "arrow-schema 56.2.0", + "datafusion-common 50.3.0", + "datafusion-datasource 50.3.0", + "datafusion-expr-common 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-expr-common 50.3.0", + "datafusion-physical-plan 50.3.0", + "itertools", + "log", ] [[package]] name = "datafusion-pruning" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac2c2498a1f134a9e11a9f5ed202a2a7d7e9774bd9249295593053ea3be999db" +checksum = "ac2c2498a1f134a9e11a9f5ed202a2a7d7e9774bd9249295593053ea3be999db" +dependencies = [ + "arrow 57.1.0", + "datafusion-common 51.0.0", + "datafusion-datasource 51.0.0", + "datafusion-expr-common 51.0.0", + "datafusion-physical-expr 51.0.0", + "datafusion-physical-expr-common 51.0.0", + "datafusion-physical-plan 51.0.0", + "itertools", + "log", +] + +[[package]] +name = "datafusion-session" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ef8e2745583619bd7a49474e8f45fbe98ebb31a133f27802217125a7b3d58d" dependencies = [ - "arrow", - "datafusion-common", - "datafusion-datasource", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", + "arrow 56.2.0", + "async-trait", + "dashmap", + "datafusion-common 50.3.0", + "datafusion-common-runtime 50.3.0", + "datafusion-execution 50.3.0", + "datafusion-expr 50.3.0", + "datafusion-physical-expr 50.3.0", + "datafusion-physical-plan 50.3.0", + "datafusion-sql 50.3.0", + "futures", "itertools", "log", + "object_store", + "parking_lot", + "tokio", ] [[package]] @@ -2337,29 +3360,46 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f96eebd17555386f459037c65ab73aae8df09f464524c709d6a3134ad4f4776" dependencies = [ "async-trait", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-plan", + "datafusion-common 51.0.0", + "datafusion-execution 51.0.0", + "datafusion-expr 51.0.0", + "datafusion-physical-plan 51.0.0", "parking_lot", ] +[[package]] +name = "datafusion-sql" +version = "50.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89abd9868770386fede29e5a4b14f49c0bf48d652c3b9d7a8a0332329b87d50b" +dependencies = [ + "arrow 56.2.0", + "bigdecimal", + "datafusion-common 50.3.0", + "datafusion-expr 50.3.0", + "indexmap", + "log", + "recursive", + "regex", + "sqlparser 0.58.0", +] + [[package]] name = "datafusion-sql" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fc195fe60634b2c6ccfd131b487de46dc30eccae8a3c35a13f136e7f440414f" dependencies = [ - "arrow", + "arrow 57.1.0", "bigdecimal", "chrono", - "datafusion-common", - "datafusion-expr", + "datafusion-common 51.0.0", + "datafusion-expr 51.0.0", "indexmap", "log", "recursive", "regex", - "sqlparser", + "sqlparser 0.59.0", ] [[package]] @@ -3480,6 +4520,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lz4_flex" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" +dependencies = [ + "twox-hash", +] + [[package]] name = "lz4_flex" version = "0.12.0" @@ -3510,6 +4559,15 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "matchit" version = "0.7.3" @@ -3591,6 +4649,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + [[package]] name = "native-tls" version = "0.2.14" @@ -3629,6 +4693,29 @@ dependencies = [ "libc", ] +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -3664,6 +4751,28 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -3830,6 +4939,43 @@ dependencies = [ "windows-link", ] +[[package]] +name = "parquet" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dbd48ad52d7dccf8ea1b90a3ddbfaea4f69878dd7683e51c507d4bc52b5b27" +dependencies = [ + "ahash", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-data 56.2.0", + "arrow-ipc 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "base64", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.16.1", + "lz4_flex 0.11.5", + "num", + "num-bigint", + "object_store", + "paste", + "ring", + "seq-macro", + "simdutf8", + "snap", + "thrift", + "tokio", + "twox-hash", + "zstd", +] + [[package]] name = "parquet" version = "57.1.0" @@ -3837,13 +4983,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be3e4f6d320dd92bfa7d612e265d7d08bba0a240bab86af3425e1d255a511d89" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-ipc", - "arrow-schema", - "arrow-select", + "arrow-array 57.1.0", + "arrow-buffer 57.1.0", + "arrow-cast 57.1.0", + "arrow-data 57.1.0", + "arrow-ipc 57.1.0", + "arrow-schema 57.1.0", + "arrow-select 57.1.0", "base64", "brotli", "bytes", @@ -3852,7 +4998,7 @@ dependencies = [ "futures", "half", "hashbrown 0.16.1", - "lz4_flex", + "lz4_flex 0.12.0", "num-bigint", "num-integer", "num-traits", @@ -3890,6 +5036,16 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "petgraph" version = "0.8.3" @@ -4065,6 +5221,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive 0.13.5", +] + [[package]] name = "prost" version = "0.14.1" @@ -4072,7 +5238,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.14.1", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck 0.5.0", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph 0.7.1", + "prettyplease", + "prost 0.13.5", + "prost-types 0.13.5", + "regex", + "syn 2.0.110", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.110", ] [[package]] @@ -4088,13 +5287,22 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost 0.13.5", +] + [[package]] name = "prost-types" version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" dependencies = [ - "prost", + "prost 0.14.1", ] [[package]] @@ -4755,6 +5963,15 @@ dependencies = [ "digest", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "1.3.0" @@ -4832,6 +6049,17 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "sqlparser" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4b661c54b1e4b603b37873a18c59920e4c51ea8ea2cf527d925424dbd4437c" +dependencies = [ + "log", + "recursive", + "sqlparser_derive", +] + [[package]] name = "sqlparser" version = "0.59.0" @@ -4909,12 +6137,31 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" + [[package]] name = "strum" version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.110", +] + [[package]] name = "strum_macros" version = "0.27.2" @@ -5044,6 +6291,15 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + [[package]] name = "thrift" version = "0.17.0" @@ -5062,6 +6318,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" dependencies = [ "deranged", + "itoa", "num-conv", "powerfmt", "serde", @@ -5231,6 +6488,35 @@ dependencies = [ "winnow", ] +[[package]] +name = "tonic" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +dependencies = [ + "async-trait", + "axum 0.8.7", + "base64", + "bytes", + "h2 0.4.12", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "hyper 1.8.1", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost 0.13.5", + "socket2 0.5.10", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tonic" version = "0.14.2" @@ -5260,6 +6546,20 @@ dependencies = [ "tracing", ] +[[package]] +name = "tonic-build" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac6f67be712d12f0b41328db3137e0d0757645d8904b4cb7d51cd9c2279e847" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types 0.13.5", + "quote", + "syn 2.0.110", +] + [[package]] name = "tonic-prost" version = "0.14.2" @@ -5267,8 +6567,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" dependencies = [ "bytes", - "prost", - "tonic", + "prost 0.14.1", + "tonic 0.14.2", ] [[package]] @@ -5330,15 +6630,15 @@ name = "tpchgen-arrow" version = "2.0.1" source = "git+https://github.com/clflushopt/tpchgen-rs?rev=e83365a5a9101906eb9f78c5607b83bc59849acf#e83365a5a9101906eb9f78c5607b83bc59849acf" dependencies = [ - "arrow", + "arrow 57.1.0", "tpchgen", ] [[package]] name = "tracing" -version = "0.1.41" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "log", "pin-project-lite", @@ -5346,11 +6646,23 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf" +dependencies = [ + "crossbeam-channel", + "thiserror", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", @@ -5359,11 +6671,41 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.34" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +dependencies = [ + "matchers", + "nu-ansi-term", "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", ] [[package]] @@ -5456,6 +6798,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 3c7394a9..e9efe685 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -8,6 +8,10 @@ default-run = "dfbench" datafusion = { workspace = true } datafusion-proto = { workspace = true } datafusion-distributed = { path = "..", features = ["integration"] } +ballista = { version = "50" } +ballista-executor = { version = "50" } +ballista-scheduler = { version = "50" } +ballista-core = "50" tokio = { version = "1.46.1", features = ["full"] } parquet = { version = "57.1.0" } structopt = { version = "0.3.26" } @@ -28,6 +32,7 @@ object_store = { version = "0.12.4", features = ["aws"] } aws-config = "1" aws-sdk-ec2 = "1" openssl = { version = "0.10", features = ["vendored"] } +clap = "4.5" [[bin]] name = "dfbench" @@ -36,3 +41,15 @@ path = "src/main.rs" [[bin]] name = "worker" path = "cdk/bin/worker.rs" + +[[bin]] +name = "ballista-http" +path = "cdk/bin/ballista_http.rs" + +[[bin]] +name = "ballista-executor" +path = "cdk/bin/ballista_executor.rs" + +[[bin]] +name = "ballista-scheduler" +path = "cdk/bin/ballista_scheduler.rs" diff --git a/benchmarks/cdk/bin/ballista-bench.ts b/benchmarks/cdk/bin/ballista-bench.ts new file mode 100644 index 00000000..02b9d440 --- /dev/null +++ b/benchmarks/cdk/bin/ballista-bench.ts @@ -0,0 +1,97 @@ +import path from "path"; +import {Command} from "commander"; +import {z} from 'zod'; +import {BenchmarkRunner, ROOT, runBenchmark, TableSpec} from "./@bench-common"; + +// Remember to port-forward the ballista HTTP server with +// aws ssm start-session --target {host-id} --document-name AWS-StartPortForwardingSession --parameters "portNumber=9002,localPortNumber=9002" + +async function main() { + const program = new Command(); + + program + .option('--dataset ', 'Dataset to run queries on') + .option('-i, --iterations ', 'Number of iterations', '3') + .option('--query ', 'A specific query to run', undefined) + .parse(process.argv); + + const options = program.opts(); + + const dataset: string = options.dataset + const iterations = parseInt(options.iterations); + const queries = options.query ? [parseInt(options.query)] : []; + + const runner = new BallistaRunner({}); + + const datasetPath = path.join(ROOT, "benchmarks", "data", dataset); + const outputPath = path.join(datasetPath, "remote-results.json") + + await runBenchmark(runner, { + dataset, + iterations, + queries, + outputPath, + }); +} + +const QueryResponse = z.object({ + count: z.number(), + plan: z.string() +}) +type QueryResponse = z.infer + +class BallistaRunner implements BenchmarkRunner { + private url = 'http://localhost:9002'; + + constructor(private readonly options: {}) { + } + + async executeQuery(sql: string): Promise<{ rowCount: number }> { + let response + if (sql.includes("create view")) { + // This is query 15 + let [createView, query, dropView] = sql.split(";") + await this.query(createView); + response = await this.query(query) + await this.query(dropView); + } else { + response = await this.query(sql) + } + + return { rowCount: response.count }; + } + + private async query(sql: string): Promise { + const url = new URL(this.url); + url.searchParams.set('sql', sql); + + const response = await fetch(url.toString()); + + if (!response.ok) { + const msg = await response.text(); + throw new Error(`Query failed: ${response.status} ${msg}`); + } + + const unparsed = await response.json(); + return QueryResponse.parse(unparsed); + } + + async createTables(tables: TableSpec[]): Promise { + let stmt = ''; + for (const table of tables) { + // language=SQL format=false + stmt += ` + DROP TABLE IF EXISTS ${table.name}; + CREATE EXTERNAL TABLE IF NOT EXISTS ${table.name} STORED AS PARQUET LOCATION '${table.s3Path}'; + `; + } + await this.query(stmt); + } + +} + +main() + .catch(err => { + console.error(err) + process.exit(1) + }) diff --git a/benchmarks/cdk/bin/ballista_executor.rs b/benchmarks/cdk/bin/ballista_executor.rs new file mode 100644 index 00000000..87f660c0 --- /dev/null +++ b/benchmarks/cdk/bin/ballista_executor.rs @@ -0,0 +1,36 @@ +use ballista::datafusion::execution::runtime_env::RuntimeEnv; +use ballista::datafusion::prelude::SessionConfig; +use ballista_executor::config::Config; +use ballista_executor::executor_process::{ExecutorProcessConfig, start_executor_process}; +use clap::Parser; +use object_store::aws::AmazonS3Builder; +use std::env; +use std::sync::Arc; +use url::Url; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let opt = Config::parse(); + + let mut config: ExecutorProcessConfig = opt.try_into()?; + + let bucket = env::var("BUCKET").unwrap_or("datafusion-distributed-benchmarks".to_string()); + let s3_url = Url::parse(&format!("s3://{bucket}"))?; + + let s3 = Arc::new( + AmazonS3Builder::from_env() + .with_bucket_name(s3_url.host().unwrap().to_string()) + .build()?, + ); + let runtime_env = Arc::new(RuntimeEnv::default()); + runtime_env.register_object_store(&s3_url, s3); + + config.override_runtime_producer = Some(Arc::new( + move |_: &SessionConfig| -> ballista::datafusion::common::Result> { + Ok(runtime_env.clone()) + }, + )); + + start_executor_process(Arc::new(config)).await?; + Ok(()) +} diff --git a/benchmarks/cdk/bin/ballista_http.rs b/benchmarks/cdk/bin/ballista_http.rs new file mode 100644 index 00000000..948aa1e1 --- /dev/null +++ b/benchmarks/cdk/bin/ballista_http.rs @@ -0,0 +1,124 @@ +use axum::{Json, Router, extract::Query, http::StatusCode, routing::get}; +use ballista::datafusion::common::instant::Instant; +use ballista::datafusion::execution::SessionStateBuilder; +use ballista::datafusion::execution::runtime_env::RuntimeEnv; +use ballista::datafusion::physical_plan::displayable; +use ballista::datafusion::physical_plan::execute_stream; +use ballista::datafusion::prelude::SessionConfig; +use ballista::datafusion::prelude::SessionContext; +use ballista::prelude::*; +use futures::{StreamExt, TryFutureExt}; +use log::{error, info}; +use object_store::aws::AmazonS3Builder; +use serde::Serialize; +use std::collections::HashMap; +use std::error::Error; +use std::fmt::Display; +use std::sync::Arc; +use structopt::StructOpt; +use url::Url; + +#[derive(Serialize)] +struct QueryResult { + plan: String, + count: usize, +} + +#[derive(Debug, StructOpt, Clone)] +#[structopt(about = "worker spawn command")] +struct Cmd { + /// The bucket name. + #[structopt(long, default_value = "datafusion-distributed-benchmarks")] + bucket: String, +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + env_logger::builder() + .filter_level(log::LevelFilter::Info) + .parse_default_env() + .init(); + + let cmd = Cmd::from_args(); + + const LISTENER_ADDR: &str = "0.0.0.0:9002"; + + info!("Starting HTTP listener on {LISTENER_ADDR}..."); + let listener = tokio::net::TcpListener::bind(LISTENER_ADDR).await?; + + // Register S3 object store + let s3_url = Url::parse(&format!("s3://{}", cmd.bucket))?; + + info!("Building shared SessionContext for the whole lifetime of the HTTP listener..."); + let s3 = Arc::new( + AmazonS3Builder::from_env() + .with_bucket_name(s3_url.host().unwrap().to_string()) + .build()?, + ); + let runtime_env = Arc::new(RuntimeEnv::default()); + runtime_env.register_object_store(&s3_url, s3); + + let config = SessionConfig::new_with_ballista().with_ballista_job_name("Benchmarks"); + + let state = SessionStateBuilder::new() + .with_config(config) + .with_default_features() + .with_runtime_env(Arc::clone(&runtime_env)) + .build(); + let ctx = SessionContext::remote_with_state("df://localhost:50050", state).await?; + + let http_server = axum::serve( + listener, + Router::new().route( + "/", + get(move |Query(params): Query>| { + let ctx = ctx.clone(); + + async move { + let sql = params.get("sql").ok_or(err("Missing 'sql' parameter"))?; + + let mut df_opt = None; + for sql in sql.split(";") { + if sql.trim().is_empty() { + continue; + } + let df = ctx.sql(sql).await.map_err(err)?; + df_opt = Some(df); + } + let Some(df) = df_opt else { + return Err(err("Empty 'sql' parameter")); + }; + + let start = Instant::now(); + + info!("Executing query..."); + let physical = df.create_physical_plan().await.map_err(err)?; + let mut stream = + execute_stream(physical.clone(), ctx.task_ctx()).map_err(err)?; + let mut count = 0; + while let Some(batch) = stream.next().await { + count += batch.map_err(err)?.num_rows(); + info!("Gathered {count} rows, query still in progress..") + } + let plan = displayable(physical.as_ref()).indent(true).to_string(); + let elapsed = start.elapsed(); + let ms = elapsed.as_secs_f64() * 1000.0; + info!("Returned {count} rows in {ms} ms"); + + Ok::<_, (StatusCode, String)>(Json(QueryResult { count, plan })) + } + .inspect_err(|(_, msg)| { + error!("Error executing query: {msg}"); + }) + }), + ), + ); + + info!("Started listener HTTP server in {LISTENER_ADDR}"); + http_server.await?; + Ok(()) +} + +fn err(s: impl Display) -> (StatusCode, String) { + (StatusCode::INTERNAL_SERVER_ERROR, s.to_string()) +} diff --git a/benchmarks/cdk/bin/ballista_scheduler.rs b/benchmarks/cdk/bin/ballista_scheduler.rs new file mode 100644 index 00000000..ba0b1f2d --- /dev/null +++ b/benchmarks/cdk/bin/ballista_scheduler.rs @@ -0,0 +1,62 @@ +use ballista::datafusion::execution::runtime_env::RuntimeEnv; +use ballista::datafusion::execution::{SessionState, SessionStateBuilder}; +use ballista::datafusion::prelude::SessionConfig; +use ballista_core::error::BallistaError; +use ballista_core::extension::SessionConfigExt; +use ballista_scheduler::cluster::BallistaCluster; +use ballista_scheduler::config::{Config, SchedulerConfig}; +use ballista_scheduler::scheduler_process::start_server; +use clap::Parser; +use object_store::aws::AmazonS3Builder; +use std::env; +use std::sync::Arc; +use url::Url; + +fn main() -> Result<(), Box> { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_io() + .enable_time() + .thread_stack_size(32 * 1024 * 1024) // 32MB + .build()?; + + runtime.block_on(inner()) +} + +async fn inner() -> Result<(), Box> { + let opt = Config::parse(); + + let addr = format!("{}:{}", opt.bind_host, opt.bind_port); + let addr = addr + .parse() + .map_err(|e: std::net::AddrParseError| BallistaError::Configuration(e.to_string()))?; + + let bucket = env::var("BUCKET").unwrap_or("datafusion-distributed-benchmarks".to_string()); + let s3_url = Url::parse(&format!("s3://{bucket}"))?; + + let s3 = Arc::new( + AmazonS3Builder::from_env() + .with_bucket_name(s3_url.host().unwrap().to_string()) + .build()?, + ); + let runtime_env = Arc::new(RuntimeEnv::default()); + runtime_env.register_object_store(&s3_url, s3); + + let config: SchedulerConfig = opt.try_into()?; + let config = config.with_override_config_producer(Arc::new(|| { + SessionConfig::new_with_ballista().with_information_schema(true) + })); + let config = config.with_override_session_builder(Arc::new( + move |cfg: SessionConfig| -> ballista::datafusion::common::Result { + Ok(SessionStateBuilder::new() + .with_config(cfg) + .with_runtime_env(runtime_env.clone()) + .with_default_features() + .build()) + }, + )); + + let cluster = BallistaCluster::new_from_config(&config).await?; + start_server(cluster, addr, Arc::new(config)).await?; + + Ok(()) +} diff --git a/benchmarks/cdk/bin/cdk.ts b/benchmarks/cdk/bin/cdk.ts index 2d1fd482..442fe003 100644 --- a/benchmarks/cdk/bin/cdk.ts +++ b/benchmarks/cdk/bin/cdk.ts @@ -1,12 +1,20 @@ #!/usr/bin/env node import * as cdk from 'aws-cdk-lib/core'; -import { CdkStack } from '../lib/cdk-stack'; +import {CdkStack} from '../lib/cdk-stack'; +import {DATAFUSION_DISTRIBUTED_ENGINE} from "../lib/datafusion-distributed"; +import {TRINO_ENGINE} from "../lib/trino"; +import {BALLISTA_ENGINE} from "../lib/ballista"; const app = new cdk.App(); const config = { - instanceType: 't3.xlarge', - instanceCount: 4, + instanceType: 't3.xlarge', + instanceCount: 4, + engines: [ + DATAFUSION_DISTRIBUTED_ENGINE, + TRINO_ENGINE, + BALLISTA_ENGINE + ] }; new CdkStack(app, 'DataFusionDistributedBenchmarks', { config }); diff --git a/benchmarks/cdk/lib/ballista.ts b/benchmarks/cdk/lib/ballista.ts new file mode 100644 index 00000000..5e138ee3 --- /dev/null +++ b/benchmarks/cdk/lib/ballista.ts @@ -0,0 +1,212 @@ +import { + AfterEc2MachinesContext, + BeforeEc2MachinesContext, + QueryEngine, + ROOT, + sendCommandsUnconditionally, + OnEc2MachinesContext +} from "./cdk-stack"; +import * as s3assets from "aws-cdk-lib/aws-s3-assets"; +import path from "path"; +import {execSync} from "child_process"; + +let ballistaServerBinary: s3assets.Asset +let ballistaSchedulerBinary: s3assets.Asset +let ballistaExecutorBinary: s3assets.Asset + +export const BALLISTA_ENGINE: QueryEngine = { + beforeEc2Machines(ctx: BeforeEc2MachinesContext): void { + console.log('Building Ballista server binary...'); + execSync('cargo zigbuild -p datafusion-distributed-benchmarks --release --bin ballista-http --target x86_64-unknown-linux-gnu', { + cwd: ROOT, + stdio: 'inherit', + }); + console.log('Ballista server binary built successfully'); + + console.log('Building Ballista scheduler...'); + execSync('cargo zigbuild -p datafusion-distributed-benchmarks --release --bin ballista-scheduler --target x86_64-unknown-linux-gnu', { + cwd: ROOT, + stdio: 'inherit', + }); + console.log('Ballista scheduler built successfully'); + + console.log('Building Ballista executor...'); + execSync('cargo zigbuild -p datafusion-distributed-benchmarks --release --bin ballista-executor --target x86_64-unknown-linux-gnu', { + cwd: ROOT, + stdio: 'inherit', + }); + console.log('Ballista scheduler built successfully'); + + ballistaServerBinary = new s3assets.Asset(ctx.scope, 'BallistaServerBinary', { + path: path.join(ROOT, 'target/x86_64-unknown-linux-gnu/release/ballista-http'), + }) + + ballistaSchedulerBinary = new s3assets.Asset(ctx.scope, 'BallistaSchedulerBinary', { + path: path.join(ROOT, 'target/x86_64-unknown-linux-gnu/release/ballista-scheduler'), + }) + + ballistaExecutorBinary = new s3assets.Asset(ctx.scope, 'BallistaExecutorBinary', { + path: path.join(ROOT, 'target/x86_64-unknown-linux-gnu/release/ballista-executor'), + }) + + ballistaServerBinary.grantRead(ctx.role) + ballistaSchedulerBinary.grantRead(ctx.role) + ballistaExecutorBinary.grantRead(ctx.role) + }, + onEc2Machine(ctx: OnEc2MachinesContext): void { + const isScheduler = ctx.instanceIdx === 0; + ctx.instanceUserData.addCommands( + // Download pre-compiled Ballista binaries from S3 + `aws s3 cp s3://${ballistaSchedulerBinary.s3BucketName}/${ballistaSchedulerBinary.s3ObjectKey} /usr/local/bin/ballista-scheduler`, + 'chmod +x /usr/local/bin/ballista-scheduler', + `aws s3 cp s3://${ballistaExecutorBinary.s3BucketName}/${ballistaExecutorBinary.s3ObjectKey} /usr/local/bin/ballista-executor`, + 'chmod +x /usr/local/bin/ballista-executor', + `aws s3 cp s3://${ballistaServerBinary.s3BucketName}/${ballistaServerBinary.s3ObjectKey} /usr/local/bin/ballista-http`, + 'chmod +x /usr/local/bin/ballista-http', + + // Create Ballista directories + 'mkdir -p /var/ballista/scheduler', + 'mkdir -p /var/ballista/executor', + 'mkdir -p /var/ballista/logs', + + // Create Ballista scheduler systemd service (coordinator only) + ...(isScheduler ? [ + `cat > /etc/systemd/system/ballista-scheduler.service << 'BALLISTA_EOF' +[Unit] +Description=Ballista Scheduler +After=network.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/ballista-scheduler \\ + --bind-host 0.0.0.0 \\ + --bind-port 50050 +Restart=on-failure +RestartSec=5 +User=root +WorkingDirectory=/var/ballista/scheduler +StandardOutput=append:/var/ballista/logs/scheduler.log +StandardError=append:/var/ballista/logs/scheduler.log + +[Install] +WantedBy=multi-user.target +BALLISTA_EOF` + ] : []), + + // Create Ballista executor systemd service (all nodes, will be reconfigured for workers) + `cat > /etc/systemd/system/ballista-executor.service << 'BALLISTA_EOF' +[Unit] +Description=Ballista Executor +After=network.target${isScheduler ? ' ballista-scheduler.service' : ''} +${isScheduler ? 'Requires=ballista-scheduler.service' : ''} + +[Service] +Type=simple +ExecStart=/usr/local/bin/ballista-executor \\ + --bind-host 0.0.0.0 \\ + --bind-port 50051 \\ + --work-dir /var/ballista/executor \\ + --scheduler-host localhost \\ + --scheduler-port 50050 +Restart=on-failure +RestartSec=5 +User=root +Environment="BUCKET=${ctx.bucketName}" +WorkingDirectory=/var/ballista/executor +StandardOutput=append:/var/ballista/logs/executor.log +StandardError=append:/var/ballista/logs/executor.log + +[Install] +WantedBy=multi-user.target +BALLISTA_EOF`, + + // Create HTTP server systemd service (coordinator only for now) + ...(isScheduler ? [ + `aws s3 cp s3://${ballistaServerBinary.s3BucketName}/${ballistaServerBinary.s3ObjectKey} /usr/local/bin/ballista-http`, + 'chmod +x /usr/local/bin/ballista-http', + `cat > /etc/systemd/system/ballista-http.service << 'BALLISTA_EOF' +[Unit] +Description=Ballista HTTP Server +After=network.target ballista-scheduler.service +Requires=ballista-scheduler.service + +[Service] +Type=simple +ExecStart=/usr/local/bin/ballista-http --bucket ${ctx.bucketName} +Restart=on-failure +RestartSec=5 +User=root +Environment="RUST_LOG=info" +WorkingDirectory=/var/ballista +StandardOutput=append:/var/ballista/logs/http.log +StandardError=append:/var/ballista/logs/http.log + +[Install] +WantedBy=multi-user.target +BALLISTA_EOF` + ] : []), + + // Reload systemd and enable services + 'systemctl daemon-reload', + + // Enable and start scheduler (coordinator only) + ...(isScheduler ? [ + 'systemctl enable ballista-scheduler', + 'systemctl start ballista-scheduler', + // Wait for scheduler to be ready + 'sleep 5' + ] : []), + + // Enable and start executor (all nodes) + 'systemctl enable ballista-executor', + 'systemctl start ballista-executor', + + // Enable and start HTTP server (coordinator only) + ...(isScheduler ? [ + 'systemctl enable ballista-http', + 'systemctl start ballista-http' + ] : []) + ) + + }, + afterEc2Machines(ctx: AfterEc2MachinesContext) { + const [scheduler, ...executors] = ctx.instances + + // Reconfigure executors on worker nodes to point to scheduler. The executor in the machine holding the scheduler + // communicates to it using localhost, so no need to update it with scheduler.instancePrivateIp. + sendCommandsUnconditionally( + ctx.scope, + "ConfigureBallistaExecutors", + [scheduler, ...executors], + [ + `cat > /etc/systemd/system/ballista-executor.service << 'BALLISTA_EOF' +[Unit] +Description=Ballista Executor +After=network.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/ballista-executor \\ + --bind-host 0.0.0.0 \\ + --bind-port 50051 \\ + --work-dir /var/ballista/executor \\ + --scheduler-host ${scheduler.instancePrivateIp} \\ + --scheduler-port 50050 +Restart=on-failure +RestartSec=5 +User=root +Environment="BUCKET=${ctx.bucketName}" +WorkingDirectory=/var/ballista/executor +StandardOutput=append:/var/ballista/logs/executor.log +StandardError=append:/var/ballista/logs/executor.log + +[Install] +WantedBy=multi-user.target +BALLISTA_EOF`, + 'systemctl daemon-reload', + 'systemctl restart ballista-executor', + ] + ) + } +} + diff --git a/benchmarks/cdk/lib/cdk-stack.ts b/benchmarks/cdk/lib/cdk-stack.ts index c821217f..febeb01d 100644 --- a/benchmarks/cdk/lib/cdk-stack.ts +++ b/benchmarks/cdk/lib/cdk-stack.ts @@ -1,165 +1,189 @@ -import { CfnOutput, RemovalPolicy, Stack, StackProps, Tags } from 'aws-cdk-lib'; +import {CfnOutput, RemovalPolicy, Stack, StackProps, Tags} from 'aws-cdk-lib'; import * as ec2 from 'aws-cdk-lib/aws-ec2'; import * as s3 from 'aws-cdk-lib/aws-s3'; import * as iam from 'aws-cdk-lib/aws-iam'; -import * as s3assets from 'aws-cdk-lib/aws-s3-assets'; -import * as cr from 'aws-cdk-lib/custom-resources'; -import { Construct } from 'constructs'; -import * as path from 'path'; -import { execSync } from 'child_process'; -import { trinoWorkerCommands, trinoUserDataCommands } from "./trino"; +import {Construct} from 'constructs'; +import {DATAFUSION_DISTRIBUTED_ENGINE} from "./datafusion-distributed"; +import {BALLISTA_ENGINE} from "./ballista"; +import {TRINO_ENGINE} from "./trino"; +import path from "path"; +import * as cr from "aws-cdk-lib/custom-resources"; + +const USER_DATA_CAUSES_REPLACEMENT = process.env['USER_DATA_CAUSES_REPLACEMENT'] == 'true' +if (USER_DATA_CAUSES_REPLACEMENT) { + console.warn("Instances will forcefully get replaced") +} -const ROOT = path.join(__dirname, '../../..') +const ENGINES = [ + DATAFUSION_DISTRIBUTED_ENGINE, + BALLISTA_ENGINE, + TRINO_ENGINE +] -interface CdkStackProps extends StackProps { - config: { - instanceType: string; - instanceCount: number; - }; +export const ROOT = path.join(__dirname, '../../..') + +export interface BeforeEc2MachinesContext { + scope: Construct + role: iam.Role } -export class CdkStack extends Stack { - constructor(scope: Construct, id: string, props: CdkStackProps) { - super(scope, id, props); - - const { config } = props; - - // Create VPC with public subnets only (for internet access without NAT gateway) - const vpc = new ec2.Vpc(this, 'BenchmarkVPC', { - maxAzs: 1, - natGateways: 0, - subnetConfiguration: [ - { - name: 'Public', - subnetType: ec2.SubnetType.PUBLIC, - cidrMask: 24, - }, - ], - }); +export interface OnEc2MachinesContext { + instanceIdx: number + instanceUserData: ec2.UserData + region: string + bucketName: string +} - // Create security group that allows instances to communicate - const securityGroup = new ec2.SecurityGroup(this, 'BenchmarkSG', { - vpc, - allowAllOutbound: true, - }); +export interface AfterEc2MachinesContext { + scope: Construct + instances: ec2.Instance[] + bucketName: string + region: string +} - // Allow all traffic between instances in the same security group - securityGroup.addIngressRule( - securityGroup, - ec2.Port.allTraffic(), - 'Allow all traffic between benchmark instances' - ); - - // Create S3 bucket - const bucket = new s3.Bucket(this, 'BenchmarkBucket', { - bucketName: "datafusion-distributed-benchmarks", - autoDeleteObjects: true, - removalPolicy: RemovalPolicy.DESTROY - }); +export interface QueryEngine { + /** Runs before instantiating any EC2 machine */ + beforeEc2Machines(ctx: BeforeEc2MachinesContext): void - // Build worker binary for Linux - console.log('Building worker binary...'); - execSync('cargo zigbuild -p datafusion-distributed-benchmarks --release --bin worker --target x86_64-unknown-linux-gnu', { - cwd: ROOT, - stdio: 'inherit', - }); - console.log('Worker binary built successfully'); + /** Runs for each instantiated EC2 machine */ + onEc2Machine(ctx: OnEc2MachinesContext): void - // Upload worker binary as an asset - const workerBinary = new s3assets.Asset(this, 'WorkerBinary', { - path: path.join(ROOT, 'target/x86_64-unknown-linux-gnu/release/worker'), - }); + /** Runs after all EC2 machines have been instantiated */ + afterEc2Machines(ctx: AfterEc2MachinesContext): void +} - // Create IAM role for EC2 instances - const role = new iam.Role(this, 'BenchmarkInstanceRole', { - assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'), - managedPolicies: [ - iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore'), - ], - }); - // Grant permissions to describe EC2 instances (for peer discovery) - role.addToPolicy(new iam.PolicyStatement({ - actions: ['ec2:DescribeInstances'], - resources: ['*'], - })); - - // Grant Glue permissions for Trino Hive metastore - role.addToPolicy(new iam.PolicyStatement({ - actions: [ - 'glue:GetDatabase', - 'glue:GetDatabases', - 'glue:GetTable', - 'glue:GetTables', - 'glue:GetPartition', - 'glue:GetPartitions', - 'glue:CreateTable', - 'glue:UpdateTable', - 'glue:DeleteTable', - 'glue:CreateDatabase', - 'glue:UpdateDatabase', - 'glue:DeleteDatabase', - ], - resources: ['*'], - })); - - // Grant read access to the bucket and worker binary - bucket.grantRead(role); - workerBinary.grantRead(role); - - // Create EC2 instances - const instances: ec2.Instance[] = []; - for (let i = 0; i < config.instanceCount; i++) { - const userData = ec2.UserData.forLinux(); - - userData.addCommands( - // Install Rust tooling. - 'yum install gcc', - "curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh", - 'cargo install --locked tokio-console', - - // Create systemd service - `cat > /etc/systemd/system/worker.service << 'EOF' -[Unit] -Description=DataFusion Distributed Worker -After=network.target - -[Service] -Type=simple -ExecStart=/usr/local/bin/worker --bucket ${bucket.bucketName} -Restart=always -User=root - -[Install] -WantedBy=multi-user.target -EOF`, - - // Enable and start the service - 'systemctl daemon-reload', - 'systemctl enable worker', - 'systemctl start worker', - ...trinoUserDataCommands(i, this.region) - ); - - const instance = new ec2.Instance(this, `BenchmarkInstance${i}`, { - vpc, - vpcSubnets: { subnetType: ec2.SubnetType.PUBLIC }, - instanceName: `instance-${i}`, - instanceType: new ec2.InstanceType(config.instanceType), - machineImage: ec2.MachineImage.latestAmazonLinux2023(), - securityGroup, - role, - userData - }); - - // Tag for peer discovery - Tags.of(instance).add('BenchmarkCluster', 'datafusion'); - instances.push(instance); - } +interface CdkStackProps extends StackProps { + config: { + instanceType: string; + instanceCount: number; + engines: QueryEngine[] + }; +} - // Output Session Manager commands for all instances - new CfnOutput(this, 'ConnectCommands', { - value: ` +export class CdkStack extends Stack { + constructor(scope: Construct, id: string, props: CdkStackProps) { + super(scope, id, props); + + const { config } = props; + + // Create VPC with public subnets only (for internet access without NAT gateway) + const vpc = new ec2.Vpc(this, 'BenchmarkVPC', { + maxAzs: 1, + natGateways: 0, + subnetConfiguration: [ + { + name: 'Public', + subnetType: ec2.SubnetType.PUBLIC, + cidrMask: 24, + }, + ], + }); + + // Create security group that allows instances to communicate + const securityGroup = new ec2.SecurityGroup(this, 'BenchmarkSG', { + vpc, + allowAllOutbound: true, + }); + + // Allow all traffic between instances in the same security group + securityGroup.addIngressRule( + securityGroup, + ec2.Port.allTraffic(), + 'Allow all traffic between benchmark instances' + ); + + // Create S3 bucket + const bucket = new s3.Bucket(this, 'BenchmarkBucket', { + bucketName: "datafusion-distributed-benchmarks", + autoDeleteObjects: true, + removalPolicy: RemovalPolicy.DESTROY + }); + + // Create IAM role for EC2 instances + const role = new iam.Role(this, 'BenchmarkInstanceRole', { + assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'), + managedPolicies: [ + iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore'), + ], + }); + + // Grant permissions to describe EC2 instances (for peer discovery) + role.addToPolicy(new iam.PolicyStatement({ + actions: ['ec2:DescribeInstances'], + resources: ['*'], + })); + + // Grant Glue permissions for Trino Hive metastore + role.addToPolicy(new iam.PolicyStatement({ + actions: [ + 'glue:GetDatabase', + 'glue:GetDatabases', + 'glue:GetTable', + 'glue:GetTables', + 'glue:GetPartition', + 'glue:GetPartitions', + 'glue:CreateTable', + 'glue:UpdateTable', + 'glue:DeleteTable', + 'glue:CreateDatabase', + 'glue:UpdateDatabase', + 'glue:DeleteDatabase', + ], + resources: ['*'], + })); + + // Grant read access to the bucket and worker binary + bucket.grantRead(role); + + for (const engine of ENGINES) { + engine.beforeEc2Machines({ + scope: this, + role + }) + } + + // Create EC2 instances + const instances: ec2.Instance[] = []; + for (let i = 0; i < config.instanceCount; i++) { + const userData = ec2.UserData.forLinux(); + + for (const engine of ENGINES) { + engine.onEc2Machine({ + bucketName: bucket.bucketName, + instanceIdx: i, + instanceUserData: userData, + region: this.region + }) + } + + const instance = new ec2.Instance(this, `BenchmarkInstance${i}`, { + vpc, + vpcSubnets: { subnetType: ec2.SubnetType.PUBLIC }, + instanceName: `instance-${i}`, + instanceType: new ec2.InstanceType(config.instanceType), + machineImage: ec2.MachineImage.latestAmazonLinux2023(), + securityGroup, + role, + userData, + userDataCausesReplacement: USER_DATA_CAUSES_REPLACEMENT, + blockDevices: [{ + deviceName: '/dev/xvda', + volume: ec2.BlockDeviceVolume.ebs(200, { + volumeType: ec2.EbsDeviceVolumeType.GP3, + deleteOnTermination: true, + }), + }], + }); + + // Tag for peer discovery + Tags.of(instance).add('BenchmarkCluster', 'datafusion'); + instances.push(instance); + } + + // Output Session Manager commands for all instances + new CfnOutput(this, 'ConnectCommands', { + value: ` # === select one instance to connect to === ${instances.map(_ => `export INSTANCE_ID=${_.instanceId}`).join("\n")} @@ -173,53 +197,51 @@ aws ssm start-session --target $INSTANCE_ID sudo journalctl -u worker.service -f -o cat `, - description: 'Session Manager commands to connect to instances', - }); - - // Downloads the latest version of the worker binary and restarts the systemd service. - // This is done instead of the userData.addS3Download() so that the instance does not need - // to restart every time a new worker binary is available. - sendCommandsUnconditionally(this, 'RestartWorkerService', instances, [ - `aws s3 cp s3://${workerBinary.s3BucketName}/${workerBinary.s3ObjectKey} /usr/local/bin/worker`, - 'chmod +x /usr/local/bin/worker', - 'systemctl restart worker', - ]) - - // Then start workers (they will discover the coordinator) - const [coordinator, ...workers] = instances - sendCommandsUnconditionally(this, 'TrinoCoordinatorCommands', [coordinator], ['systemctl start trino']) - sendCommandsUnconditionally(this, 'TrinoWorkerCommands', workers, trinoWorkerCommands(coordinator)) - } + description: 'Session Manager commands to connect to instances', + }); + + for (const engine of ENGINES) { + engine.afterEc2Machines({ + scope: this, + instances, + region: this.region, + bucketName: bucket.bucketName + }) + } + } } -function sendCommandsUnconditionally( - construct: Construct, - name: string, - instances: ec2.Instance[], - commands: string[] +export function sendCommandsUnconditionally( + construct: Construct, + name: string, + instances: ec2.Instance[], + commands: string[] ) { - const cmd = new cr.AwsCustomResource(construct, name, { - onUpdate: { - service: 'SSM', - action: 'sendCommand', - parameters: { - DocumentName: 'AWS-RunShellScript', - InstanceIds: instances.map(inst => inst.instanceId), - Parameters: { - commands + const cmd = new cr.AwsCustomResource(construct, name, { + onUpdate: { + service: 'SSM', + action: 'sendCommand', + parameters: { + DocumentName: 'AWS-RunShellScript', + InstanceIds: instances.map(inst => inst.instanceId), + Parameters: { + commands: [ + 'cloud-init status --wait', + ...commands + ] + }, + }, + physicalResourceId: cr.PhysicalResourceId.of(`${name}-${Date.now()}`), + ignoreErrorCodesMatching: '.*', }, - }, - physicalResourceId: cr.PhysicalResourceId.of(`${name}-${Date.now()}`), - ignoreErrorCodesMatching: '.*', - }, - policy: cr.AwsCustomResourcePolicy.fromStatements([ - new iam.PolicyStatement({ - actions: ['ssm:SendCommand'], - resources: ['*'], - }), - ]), - }); - - // Ensure instances are created before restarting - cmd.node.addDependency(...instances) + policy: cr.AwsCustomResourcePolicy.fromStatements([ + new iam.PolicyStatement({ + actions: ['ssm:SendCommand'], + resources: ['*'], + }), + ]), + }); + + // Ensure instances are created before restarting + cmd.node.addDependency(...instances) } diff --git a/benchmarks/cdk/lib/datafusion-distributed.ts b/benchmarks/cdk/lib/datafusion-distributed.ts new file mode 100644 index 00000000..9278c42b --- /dev/null +++ b/benchmarks/cdk/lib/datafusion-distributed.ts @@ -0,0 +1,73 @@ +import { + AfterEc2MachinesContext, + BeforeEc2MachinesContext, + OnEc2MachinesContext, + QueryEngine, + ROOT, + sendCommandsUnconditionally +} from "./cdk-stack"; +import {execSync} from "child_process"; +import * as s3assets from "aws-cdk-lib/aws-s3-assets"; +import path from "path"; + +let workerBinary: s3assets.Asset + +export const DATAFUSION_DISTRIBUTED_ENGINE: QueryEngine = { + beforeEc2Machines(ctx: BeforeEc2MachinesContext): void { + console.log('Building worker binary...'); + execSync('cargo zigbuild -p datafusion-distributed-benchmarks --release --bin worker --target x86_64-unknown-linux-gnu', { + cwd: ROOT, + stdio: 'inherit', + }); + console.log('Worker binary built successfully'); + + + // Upload worker binary as an asset + workerBinary = new s3assets.Asset(ctx.scope, 'WorkerBinary', { + path: path.join(ROOT, 'target/x86_64-unknown-linux-gnu/release/worker'), + }); + + workerBinary.grantRead(ctx.role) + }, + onEc2Machine(ctx: OnEc2MachinesContext): void { + ctx.instanceUserData.addCommands( + 'yum install -y gcc openssl-devel', + 'curl --proto \'=https\' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y', + + `aws s3 cp s3://${workerBinary.s3BucketName}/${workerBinary.s3ObjectKey} /usr/local/bin/worker`, + 'chmod +x /usr/local/bin/worker', + // Create systemd service + `cat > /etc/systemd/system/worker.service << 'EOF' +[Unit] +Description=DataFusion Distributed Worker +After=network.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/worker --bucket ${ctx.bucketName} +Restart=always +User=root + +[Install] +WantedBy=multi-user.target +EOF`, + + // Enable and start the service + 'systemctl daemon-reload', + 'systemctl enable worker', + 'systemctl start worker', + ) + }, + afterEc2Machines(ctx: AfterEc2MachinesContext): void { + // Downloads the latest version of the worker binary and restarts the systemd service. + // This is done instead of the userData.addS3Download() so that the instance does not need + // to restart every time a new worker binary is available. + sendCommandsUnconditionally(ctx.scope, 'RestartWorkerService', + ctx.instances, + [ + `aws s3 cp s3://${workerBinary.s3BucketName}/${workerBinary.s3ObjectKey} /usr/local/bin/worker`, + 'chmod +x /usr/local/bin/worker', + 'systemctl restart worker', + ]) + } +} \ No newline at end of file diff --git a/benchmarks/cdk/lib/trino.ts b/benchmarks/cdk/lib/trino.ts index ee97a45e..9614beb9 100644 --- a/benchmarks/cdk/lib/trino.ts +++ b/benchmarks/cdk/lib/trino.ts @@ -1,34 +1,42 @@ -import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import { + AfterEc2MachinesContext, + QueryEngine, + OnEc2MachinesContext, + sendCommandsUnconditionally, +} from "./cdk-stack"; const TRINO_VERSION = 476 -export function trinoUserDataCommands(instanceIndex: number, region: string): string[] { - const isCoordinator = instanceIndex === 0; - - return [ - // Install Java 24 for Trino (Trino 478 requires Java 24+) - 'yum install -y java-24-amazon-corretto-headless python', - - // Download and install Trino 478 (latest version) - 'cd /opt', - `curl -L -o trino-server.tar.gz https://repo1.maven.org/maven2/io/trino/trino-server/${TRINO_VERSION}/trino-server-${TRINO_VERSION}.tar.gz`, - 'tar -xzf trino-server.tar.gz', - `mv trino-server-${TRINO_VERSION} trino-server`, - 'rm trino-server.tar.gz', - - // Create Trino directories - 'mkdir -p /var/trino/data', - 'mkdir -p /opt/trino-server/etc/catalog', - - // Configure Trino node properties - `cat > /opt/trino-server/etc/node.properties << 'TRINO_EOF' +export const TRINO_ENGINE: QueryEngine = { + beforeEc2Machines(): void { + // nothing to compile here + }, + onEc2Machine(ctx: OnEc2MachinesContext): void { + const isCoordinator = ctx.instanceIdx === 0; + ctx.instanceUserData.addCommands( + // Install Java 24 for Trino (Trino 478 requires Java 24+) + 'yum install -y java-24-amazon-corretto-headless python', + + // Download and install Trino 478 (latest version) + 'cd /opt', + `curl -L -o trino-server.tar.gz https://repo1.maven.org/maven2/io/trino/trino-server/${TRINO_VERSION}/trino-server-${TRINO_VERSION}.tar.gz`, + 'tar -xzf trino-server.tar.gz', + `mv trino-server-${TRINO_VERSION} trino-server`, + 'rm trino-server.tar.gz', + + // Create Trino directories + 'mkdir -p /var/trino/data', + 'mkdir -p /opt/trino-server/etc/catalog', + + // Configure Trino node properties + `cat > /opt/trino-server/etc/node.properties << 'TRINO_EOF' node.environment=benchmark -node.id=instance-${instanceIndex} +node.id=instance-${ctx.instanceIdx} node.data-dir=/var/trino/data TRINO_EOF`, - // Configure Trino JVM settings (minimal - using conservative 8GB heap) - `cat > /opt/trino-server/etc/jvm.config << 'TRINO_EOF' + // Configure Trino JVM settings (minimal - using conservative 8GB heap) + `cat > /opt/trino-server/etc/jvm.config << 'TRINO_EOF' -server -Xmx8G -XX:+UseG1GC @@ -39,40 +47,40 @@ TRINO_EOF`, -Djdk.attach.allowAttachSelf=true TRINO_EOF`, - // Configure Trino config.properties (workers will be reconfigured during lazy startup) - isCoordinator - ? `cat > /opt/trino-server/etc/config.properties << 'TRINO_EOF' + // Configure Trino config.properties (workers will be reconfigured during lazy startup) + isCoordinator + ? `cat > /opt/trino-server/etc/config.properties << 'TRINO_EOF' coordinator=true node-scheduler.include-coordinator=true http-server.http.port=8080 discovery.uri=http://localhost:8080 TRINO_EOF` - : `cat > /opt/trino-server/etc/config.properties << 'TRINO_EOF' + : `cat > /opt/trino-server/etc/config.properties << 'TRINO_EOF' coordinator=false http-server.http.port=8080 discovery.uri=http://localhost:8080 TRINO_EOF`, - // Configure Hive catalog with AWS Glue metastore - `cat > /opt/trino-server/etc/catalog/hive.properties << 'TRINO_EOF' + // Configure Hive catalog with AWS Glue metastore + `cat > /opt/trino-server/etc/catalog/hive.properties << 'TRINO_EOF' connector.name=hive hive.metastore=glue -hive.metastore.glue.region=${region} +hive.metastore.glue.region=${ctx.region} fs.native-s3.enabled=true -s3.region=${region} +s3.region=${ctx.region} TRINO_EOF`, - // Configure TPCH catalog for reference - `cat > /opt/trino-server/etc/catalog/tpch.properties << 'TRINO_EOF' + // Configure TPCH catalog for reference + `cat > /opt/trino-server/etc/catalog/tpch.properties << 'TRINO_EOF' connector.name=tpch TRINO_EOF`, - // Download Trino CLI - 'curl -L -o /usr/local/bin/trino https://repo1.maven.org/maven2/io/trino/trino-cli/478/trino-cli-478-executable.jar', - 'chmod +x /usr/local/bin/trino', + // Download Trino CLI + 'curl -L -o /usr/local/bin/trino https://repo1.maven.org/maven2/io/trino/trino-cli/478/trino-cli-478-executable.jar', + 'chmod +x /usr/local/bin/trino', - // Create Trino systemd service - `cat > /etc/systemd/system/trino.service << 'TRINO_EOF' + // Create Trino systemd service + `cat > /etc/systemd/system/trino.service << 'TRINO_EOF' [Unit] Description=Trino Server After=network.target @@ -89,20 +97,29 @@ WorkingDirectory=/opt/trino-server WantedBy=multi-user.target TRINO_EOF`, - // Enable Trino (but don't start yet - will be started lazily after all instances are up) - 'systemctl daemon-reload', - 'systemctl enable trino', - 'systemctl start trino' - ]; -} - -export function trinoWorkerCommands(coordinator: ec2.Instance) { - return [ - `cat > /opt/trino-server/etc/config.properties << TRINO_EOF + // Enable Trino (but don't start yet - will be started lazily after all instances are up) + 'systemctl daemon-reload', + 'systemctl enable trino', + 'systemctl start trino' + ) + }, + afterEc2Machines(ctx: AfterEc2MachinesContext): void { + const [coordinator, ...workers] = ctx.instances + // Then start workers (they will discover the coordinator) + sendCommandsUnconditionally(ctx.scope, 'TrinoCoordinatorCommands', + [coordinator], + ['systemctl start trino'] + ) + sendCommandsUnconditionally(ctx.scope, 'TrinoWorkerCommands', + workers, + [ + `cat > /opt/trino-server/etc/config.properties << TRINO_EOF coordinator=false http-server.http.port=8080 discovery.uri=http://${coordinator.instancePrivateIp}:8080 TRINO_EOF`, - 'systemctl restart trino', - ] + 'systemctl restart trino', + ] + ) + } } \ No newline at end of file diff --git a/benchmarks/cdk/package.json b/benchmarks/cdk/package.json index 3c006f89..67b6ee81 100644 --- a/benchmarks/cdk/package.json +++ b/benchmarks/cdk/package.json @@ -11,6 +11,7 @@ "cdk": "cdk", "sync-bucket": "aws s3 sync ../data s3://datafusion-distributed-benchmarks/", "datafusion-bench": "npx ts-node bin/datafusion-bench.ts", + "ballista-bench": "npx ts-node bin/ballista-bench.ts", "trino-bench": "npx ts-node bin/trino-bench.ts" }, "devDependencies": { From a89de5f94f026c463bb0b04d93ec7ba081753380 Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Wed, 7 Jan 2026 08:55:21 +0100 Subject: [PATCH 15/27] Add Spark to remote benchmarks (#280) * Add ballista to benchmarks * Add Spark to benchmarks --- benchmarks/cdk/bin/spark-bench.ts | 109 ++++++++++ benchmarks/cdk/bin/spark_http.py | 60 ++++++ benchmarks/cdk/lib/cdk-stack.ts | 4 +- benchmarks/cdk/lib/spark.ts | 342 ++++++++++++++++++++++++++++++ benchmarks/cdk/package.json | 3 +- benchmarks/cdk/requirements.txt | 2 + 6 files changed, 518 insertions(+), 2 deletions(-) create mode 100644 benchmarks/cdk/bin/spark-bench.ts create mode 100644 benchmarks/cdk/bin/spark_http.py create mode 100644 benchmarks/cdk/lib/spark.ts create mode 100644 benchmarks/cdk/requirements.txt diff --git a/benchmarks/cdk/bin/spark-bench.ts b/benchmarks/cdk/bin/spark-bench.ts new file mode 100644 index 00000000..2588e0df --- /dev/null +++ b/benchmarks/cdk/bin/spark-bench.ts @@ -0,0 +1,109 @@ +import path from "path"; +import {Command} from "commander"; +import {z} from 'zod'; +import {BenchmarkRunner, ROOT, runBenchmark, TableSpec} from "./@bench-common"; + +// Remember to port-forward the Spark HTTP server with +// aws ssm start-session --target {host-id} --document-name AWS-StartPortForwardingSession --parameters "portNumber=9003,localPortNumber=9003" + +async function main() { + const program = new Command(); + + program + .option('--dataset ', 'Dataset to run queries on') + .option('-i, --iterations ', 'Number of iterations', '3') + .option('--query ', 'A specific query to run', undefined) + .parse(process.argv); + + const options = program.opts(); + + const dataset: string = options.dataset + const iterations = parseInt(options.iterations); + const queries = options.query ? [parseInt(options.query)] : []; + + const runner = new SparkRunner({}); + + const datasetPath = path.join(ROOT, "benchmarks", "data", dataset); + const outputPath = path.join(datasetPath, "remote-results.json") + + await runBenchmark(runner, { + dataset, + iterations, + queries, + outputPath, + }); +} + +const QueryResponse = z.object({ + count: z.number() +}) +type QueryResponse = z.infer + +class SparkRunner implements BenchmarkRunner { + private url = 'http://localhost:9003'; + + constructor(private readonly options: {}) { + } + + async executeQuery(sql: string): Promise<{ rowCount: number }> { + // Fix TPCH query 4: Add DATE prefix to date literals + sql = sql.replace(/(? { + const response = await fetch(`${this.url}/query`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + query: sql.trim().replace(/;+$/, '') + }) + }); + + if (!response.ok) { + const msg = await response.text(); + throw new Error(`Query failed: ${response.status} ${msg}`); + } + + return QueryResponse.parse(await response.json()); + } + + async createTables(tables: TableSpec[]): Promise { + for (const table of tables) { + // Spark requires s3a:// protocol, not s3:// + const s3aPath = table.s3Path.replace('s3://', 's3a://'); + + // Create temporary view from Parquet files + const createViewStmt = ` + CREATE OR REPLACE TEMPORARY VIEW ${table.name} + USING parquet + OPTIONS (path '${s3aPath}') + `; + await this.query(createViewStmt); + } + } + +} + +main() + .catch(err => { + console.error(err) + process.exit(1) + }) diff --git a/benchmarks/cdk/bin/spark_http.py b/benchmarks/cdk/bin/spark_http.py new file mode 100644 index 00000000..1b398173 --- /dev/null +++ b/benchmarks/cdk/bin/spark_http.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +import os +from flask import Flask, request, jsonify +from pyspark.sql import SparkSession + +app = Flask(__name__) + +# Initialize Spark session +spark = None + +def get_spark(): + global spark + if spark is None: + master_host = os.environ.get('SPARK_MASTER_HOST', 'localhost') + spark_jars = os.environ.get('SPARK_JARS', '/opt/spark/jars/hadoop-aws-3.4.1.jar,/opt/spark/jars/bundle-2.29.52.jar,/opt/spark/jars/aws-java-sdk-bundle-1.12.262.jar') + spark = SparkSession.builder \ + .appName("SparkHTTPServer") \ + .master(f"spark://{master_host}:7077") \ + .config("spark.jars", spark_jars) \ + .config("spark.sql.catalogImplementation", "hive") \ + .config("spark.hadoop.fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem") \ + .config("spark.hadoop.fs.s3a.aws.credentials.provider", "com.amazonaws.auth.InstanceProfileCredentialsProvider") \ + .enableHiveSupport() \ + .getOrCreate() + + # Set log level to reduce noise + spark.sparkContext.setLogLevel("WARN") + return spark + +@app.route('/health', methods=['GET']) +def health(): + """Health check endpoint""" + return jsonify({"status": "healthy"}), 200 + +@app.route('/query', methods=['POST']) +def execute_query(): + """Execute a SQL query on Spark""" + try: + data = request.get_json() + if not data or 'query' not in data: + return jsonify({"error": "Missing 'query' in request body"}), 400 + + query = data['query'] + + # Execute the query + spark_session = get_spark() + df = spark_session.sql(query) + + # Get row count without collecting all data + count = df.count() + + return jsonify({"count": count}), 200 + + except Exception as e: + return str(e), 500 + +if __name__ == '__main__': + # Run Flask server on port 9000 + port = int(os.environ.get('HTTP_PORT', 9003)) + app.run(host='0.0.0.0', port=port, debug=False) diff --git a/benchmarks/cdk/lib/cdk-stack.ts b/benchmarks/cdk/lib/cdk-stack.ts index febeb01d..f20ad86f 100644 --- a/benchmarks/cdk/lib/cdk-stack.ts +++ b/benchmarks/cdk/lib/cdk-stack.ts @@ -6,6 +6,7 @@ import {Construct} from 'constructs'; import {DATAFUSION_DISTRIBUTED_ENGINE} from "./datafusion-distributed"; import {BALLISTA_ENGINE} from "./ballista"; import {TRINO_ENGINE} from "./trino"; +import {SPARK_ENGINE} from "./spark"; import path from "path"; import * as cr from "aws-cdk-lib/custom-resources"; @@ -17,7 +18,8 @@ if (USER_DATA_CAUSES_REPLACEMENT) { const ENGINES = [ DATAFUSION_DISTRIBUTED_ENGINE, BALLISTA_ENGINE, - TRINO_ENGINE + TRINO_ENGINE, + SPARK_ENGINE ] export const ROOT = path.join(__dirname, '../../..') diff --git a/benchmarks/cdk/lib/spark.ts b/benchmarks/cdk/lib/spark.ts new file mode 100644 index 00000000..bf8a759f --- /dev/null +++ b/benchmarks/cdk/lib/spark.ts @@ -0,0 +1,342 @@ +import { + AfterEc2MachinesContext, + BeforeEc2MachinesContext, + QueryEngine, + ROOT, + sendCommandsUnconditionally, + OnEc2MachinesContext +} from "./cdk-stack"; +import * as s3assets from "aws-cdk-lib/aws-s3-assets"; +import path from "path"; + +const SPARK_VERSION = "4.0.0" +const HADOOP_VERSION = "3" + +let sparkHttpScript: s3assets.Asset +let sparkRequirements: s3assets.Asset + +export const SPARK_ENGINE: QueryEngine = { + beforeEc2Machines(ctx: BeforeEc2MachinesContext): void { + // Upload Python HTTP server script to S3 + sparkHttpScript = new s3assets.Asset(ctx.scope, 'SparkHttpScript', { + path: path.join(ROOT, 'benchmarks/cdk/bin/spark_http.py'), + }) + + // Upload Python requirements file to S3 + sparkRequirements = new s3assets.Asset(ctx.scope, 'SparkRequirements', { + path: path.join(ROOT, 'benchmarks/cdk/requirements.txt'), + }) + + sparkHttpScript.grantRead(ctx.role) + sparkRequirements.grantRead(ctx.role) + }, + onEc2Machine(ctx: OnEc2MachinesContext): void { + const isMaster = ctx.instanceIdx === 0; + ctx.instanceUserData.addCommands( + // Install Java 17 for Spark + 'yum install -y java-17-amazon-corretto-headless python3 python3-pip', + + // Download and install Spark + 'cd /opt', + `curl -L -o spark.tgz https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz`, + 'tar -xzf spark.tgz', + `mv spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark`, + 'rm spark.tgz', + + // Download AWS JARs for S3 access (Hadoop 3.4.1 with AWS SDK V2 for Spark 4.0) + 'cd /opt/spark/jars', + 'curl -L -O https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.4.1/hadoop-aws-3.4.1.jar', + 'curl -L -O https://repo1.maven.org/maven2/software/amazon/awssdk/bundle/2.29.52/bundle-2.29.52.jar', + 'curl -L -O https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.262/aws-java-sdk-bundle-1.12.262.jar', + + // Create Spark directories + 'mkdir -p /var/spark/logs', + 'mkdir -p /var/spark/work', + 'mkdir -p /var/spark/http', + + // Download Python scripts from S3 + `aws s3 cp s3://${sparkHttpScript.s3BucketName}/${sparkHttpScript.s3ObjectKey} /var/spark/http/spark_http.py`, + 'chmod +x /var/spark/http/spark_http.py', + `aws s3 cp s3://${sparkRequirements.s3BucketName}/${sparkRequirements.s3ObjectKey} /var/spark/http/requirements.txt`, + + // Install Python dependencies + 'pip3 install -r /var/spark/http/requirements.txt', + + // Configure Spark defaults + `cat > /opt/spark/conf/spark-defaults.conf << 'SPARK_EOF' +spark.master spark://localhost:7077 +spark.executor.memory 4g +spark.driver.memory 2g +spark.sql.warehouse.dir /var/spark/warehouse +spark.jars /opt/spark/jars/hadoop-aws-3.4.1.jar,/opt/spark/jars/bundle-2.29.52.jar,/opt/spark/jars/aws-java-sdk-bundle-1.12.262.jar +spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem +spark.hadoop.fs.s3a.aws.credentials.provider com.amazonaws.auth.InstanceProfileCredentialsProvider +spark.hadoop.fs.s3a.connection.timeout 60000 +spark.hadoop.fs.s3a.connection.establish.timeout 60000 +spark.hadoop.fs.s3a.attempts.maximum 10 +spark.sql.catalogImplementation hive +spark.hadoop.hive.metastore.client.factory.class com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory +spark.sql.hive.metastore.version 2.3.9 +spark.sql.hive.metastore.jars builtin +SPARK_EOF`, + + // Configure Hadoop core-site.xml for S3A with numeric timeouts + `cat > /opt/spark/conf/core-site.xml << 'CORE_SITE_EOF' + + + + fs.s3a.impl + org.apache.hadoop.fs.s3a.S3AFileSystem + + + fs.s3a.aws.credentials.provider + com.amazonaws.auth.InstanceProfileCredentialsProvider + + + fs.s3a.connection.timeout + 60000 + + + fs.s3a.connection.establish.timeout + 60000 + + + fs.s3a.connection.request.timeout + 60000 + + + fs.s3a.attempts.maximum + 10 + + + fs.s3a.retry.interval + 500 + + + fs.s3a.retry.limit + 10 + + +CORE_SITE_EOF`, + + // Configure Spark environment + `cat > /opt/spark/conf/spark-env.sh << 'SPARK_EOF' +#!/usr/bin/env bash +export SPARK_WORKER_DIR=/var/spark/work +export SPARK_LOG_DIR=/var/spark/logs +export SPARK_MASTER_HOST=localhost +export SPARK_MASTER_PORT=7077 +export SPARK_MASTER_WEBUI_PORT=8082 +export SPARK_WORKER_WEBUI_PORT=8083 +export JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64 +SPARK_EOF`, + 'chmod +x /opt/spark/conf/spark-env.sh', + + // Create Spark master systemd service (master only) + ...(isMaster ? [ + `cat > /etc/systemd/system/spark-master.service << 'SPARK_EOF' +[Unit] +Description=Spark Master +After=network.target + +[Service] +Type=forking +ExecStart=/opt/spark/sbin/start-master.sh +ExecStop=/opt/spark/sbin/stop-master.sh +Restart=on-failure +RestartSec=5 +User=root +WorkingDirectory=/opt/spark +Environment="JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64" +Environment="SPARK_LOG_DIR=/var/spark/logs" + +[Install] +WantedBy=multi-user.target +SPARK_EOF` + ] : []), + + // Create Spark worker systemd service (will be reconfigured for workers) + `cat > /etc/systemd/system/spark-worker.service << 'SPARK_EOF' +[Unit] +Description=Spark Worker +After=network.target${isMaster ? ' spark-master.service' : ''} +${isMaster ? 'Requires=spark-master.service' : ''} + +[Service] +Type=forking +ExecStart=/opt/spark/sbin/start-worker.sh spark://localhost:7077 +ExecStop=/opt/spark/sbin/stop-worker.sh +Restart=on-failure +RestartSec=5 +User=root +WorkingDirectory=/opt/spark +Environment="JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64" +Environment="SPARK_LOG_DIR=/var/spark/logs" +Environment="SPARK_WORKER_DIR=/var/spark/work" + +[Install] +WantedBy=multi-user.target +SPARK_EOF`, + + // Create HTTP server systemd service (master only) + ...(isMaster ? [ + `cat > /etc/systemd/system/spark-http.service << 'SPARK_EOF' +[Unit] +Description=Spark HTTP Server +After=network.target spark-master.service +Requires=spark-master.service + +[Service] +Type=simple +ExecStart=/usr/bin/python3 /var/spark/http/spark_http.py +Restart=on-failure +RestartSec=5 +User=root +WorkingDirectory=/var/spark/http +Environment="SPARK_MASTER_HOST=localhost" +Environment="HTTP_PORT=9003" +Environment="JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64" +Environment="SPARK_JARS=/opt/spark/jars/hadoop-aws-3.4.1.jar,/opt/spark/jars/bundle-2.29.52.jar,/opt/spark/jars/aws-java-sdk-bundle-1.12.262.jar" +StandardOutput=append:/var/spark/logs/http.log +StandardError=append:/var/spark/logs/http.log + +[Install] +WantedBy=multi-user.target +SPARK_EOF` + ] : []), + + // Reload systemd and enable services + 'systemctl daemon-reload', + + // Enable and start master (master only) + ...(isMaster ? [ + 'systemctl enable spark-master', + 'systemctl start spark-master', + 'sleep 5' + ] : []), + + // Enable and start worker (all nodes) + 'systemctl enable spark-worker', + 'systemctl start spark-worker', + + // Enable and start HTTP server (master only) + ...(isMaster ? [ + 'systemctl enable spark-http', + 'systemctl start spark-http' + ] : []) + ) + }, + afterEc2Machines(ctx: AfterEc2MachinesContext): void { + const [master, ...workers] = ctx.instances + + // Reconfigure all workers to point to master IP + sendCommandsUnconditionally( + ctx.scope, + 'ConfigureSparkWorkers', + workers, + [ + `cat > /opt/spark/conf/spark-env.sh << 'SPARK_EOF' +#!/usr/bin/env bash +export SPARK_WORKER_DIR=/var/spark/work +export SPARK_LOG_DIR=/var/spark/logs +export SPARK_MASTER_HOST=${master.instancePrivateIp} +export SPARK_MASTER_PORT=7077 +export SPARK_MASTER_WEBUI_PORT=8082 +export SPARK_WORKER_WEBUI_PORT=8081 +export JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64 +SPARK_EOF`, + 'chmod +x /opt/spark/conf/spark-env.sh', + `cat > /etc/systemd/system/spark-worker.service << 'SPARK_EOF' +[Unit] +Description=Spark Worker +After=network.target + +[Service] +Type=forking +ExecStart=/opt/spark/sbin/start-worker.sh spark://${master.instancePrivateIp}:7077 +ExecStop=/opt/spark/sbin/stop-worker.sh +Restart=on-failure +RestartSec=5 +User=root +WorkingDirectory=/opt/spark +Environment="JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64" +Environment="SPARK_LOG_DIR=/var/spark/logs" +Environment="SPARK_WORKER_DIR=/var/spark/work" + +[Install] +WantedBy=multi-user.target +SPARK_EOF`, + 'systemctl daemon-reload', + 'systemctl restart spark-worker' + ] + ) + + // Also update master's spark-env.sh to use its own IP for proper cluster formation + sendCommandsUnconditionally( + ctx.scope, + 'ConfigureSparkMaster', + [master], + [ + `cat > /opt/spark/conf/spark-env.sh << 'SPARK_EOF' +#!/usr/bin/env bash +export SPARK_WORKER_DIR=/var/spark/work +export SPARK_LOG_DIR=/var/spark/logs +export SPARK_MASTER_HOST=${master.instancePrivateIp} +export SPARK_MASTER_PORT=7077 +export SPARK_MASTER_WEBUI_PORT=8082 +export SPARK_WORKER_WEBUI_PORT=8081 +export JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64 +SPARK_EOF`, + 'chmod +x /opt/spark/conf/spark-env.sh', + `cat > /etc/systemd/system/spark-worker.service << 'SPARK_EOF' +[Unit] +Description=Spark Worker +After=network.target spark-master.service +Requires=spark-master.service + +[Service] +Type=forking +ExecStart=/opt/spark/sbin/start-worker.sh spark://${master.instancePrivateIp}:7077 +ExecStop=/opt/spark/sbin/stop-worker.sh +Restart=on-failure +RestartSec=5 +User=root +WorkingDirectory=/opt/spark +Environment="JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64" +Environment="SPARK_LOG_DIR=/var/spark/logs" +Environment="SPARK_WORKER_DIR=/var/spark/work" + +[Install] +WantedBy=multi-user.target +SPARK_EOF`, + `cat > /etc/systemd/system/spark-http.service << 'SPARK_EOF' +[Unit] +Description=Spark HTTP Server +After=network.target spark-master.service +Requires=spark-master.service + +[Service] +Type=simple +ExecStart=/usr/bin/python3 /var/spark/http/spark_http.py +Restart=on-failure +RestartSec=5 +User=root +WorkingDirectory=/var/spark/http +Environment="SPARK_MASTER_HOST=${master.instancePrivateIp}" +Environment="HTTP_PORT=9003" +Environment="JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64" +Environment="SPARK_JARS=/opt/spark/jars/hadoop-aws-3.4.1.jar,/opt/spark/jars/bundle-2.29.52.jar,/opt/spark/jars/aws-java-sdk-bundle-1.12.262.jar" +StandardOutput=append:/var/spark/logs/http.log +StandardError=append:/var/spark/logs/http.log + +[Install] +WantedBy=multi-user.target +SPARK_EOF`, + 'systemctl daemon-reload', + 'systemctl restart spark-master', + 'systemctl restart spark-worker', + 'systemctl restart spark-http' + ] + ) + } +} diff --git a/benchmarks/cdk/package.json b/benchmarks/cdk/package.json index 67b6ee81..719d58d1 100644 --- a/benchmarks/cdk/package.json +++ b/benchmarks/cdk/package.json @@ -12,7 +12,8 @@ "sync-bucket": "aws s3 sync ../data s3://datafusion-distributed-benchmarks/", "datafusion-bench": "npx ts-node bin/datafusion-bench.ts", "ballista-bench": "npx ts-node bin/ballista-bench.ts", - "trino-bench": "npx ts-node bin/trino-bench.ts" + "trino-bench": "npx ts-node bin/trino-bench.ts", + "spark-bench": "npx ts-node bin/spark-bench.ts" }, "devDependencies": { "@types/jest": "^29.5.14", diff --git a/benchmarks/cdk/requirements.txt b/benchmarks/cdk/requirements.txt new file mode 100644 index 00000000..1ee67330 --- /dev/null +++ b/benchmarks/cdk/requirements.txt @@ -0,0 +1,2 @@ +flask==3.0.0 +pyspark==4.0.0 From a938440a0ba93800dbed45a47fbb74e5b2794bed Mon Sep 17 00:00:00 2001 From: Justin O'Dwyer Date: Mon, 12 Jan 2026 08:57:47 -0500 Subject: [PATCH 16/27] Doc fixes (#288) * Doc fixes. * Minor additional changes. --- examples/in_memory.md | 2 +- src/execution_plans/network_coalesce.rs | 16 ++++++++-------- src/execution_plans/network_shuffle.rs | 22 +++++++++++----------- src/protobuf/distributed_codec.rs | 2 +- src/stage.rs | 8 ++++---- tests/tpch_correctness_test.rs | 4 ++-- tests/tpch_explain_analyze.rs | 4 ++-- tests/tpch_plans_test.rs | 3 +-- 8 files changed, 30 insertions(+), 31 deletions(-) diff --git a/examples/in_memory.md b/examples/in_memory.md index 69e7ff4b..aea65caa 100644 --- a/examples/in_memory.md +++ b/examples/in_memory.md @@ -13,7 +13,7 @@ This example queries a couple of test parquet we have for integration tests, and so pulling the first is necessary. ```shell -git intall checkout +git install checkout git lfs checkout ``` diff --git a/src/execution_plans/network_coalesce.rs b/src/execution_plans/network_coalesce.rs index 43200876..bafc09d7 100644 --- a/src/execution_plans/network_coalesce.rs +++ b/src/execution_plans/network_coalesce.rs @@ -58,11 +58,11 @@ use uuid::Uuid; /// The communication between two stages across a [NetworkCoalesceExec] has two implications: /// /// - Stage N+1 must have exactly 1 task. The distributed planner ensures this is true. -/// - The amount of partitions in the single task of Stage N+1 is equal to the sum of all -/// partitions in all tasks in Stage N+1 (e.g. (1,2,3,4,5,6,7,8,9) = (1,2,3)+(4,5,6)+(7,8,9) ) +/// - The number of partitions in the single task of Stage N+1 is equal to the total number of +/// partitions across all tasks in Stage N (e.g. (1,2,3,4,5,6,7,8,9) = (1,2,3)+(4,5,6)+(7,8,9) ) /// /// This node has two variants. -/// 1. Pending: it acts as a placeholder for the distributed optimization step to mark it as ready. +/// 1. Pending: acts as a placeholder for the distributed optimization step to mark it as ready. /// 2. Ready: runs within a distributed stage and queries the next input stage over the network /// using Arrow Flight. #[derive(Debug, Clone)] @@ -70,9 +70,9 @@ pub struct NetworkCoalesceExec { /// the properties we advertise for this execution plan pub(crate) properties: PlanProperties, pub(crate) input_stage: Stage, - /// metrics_collection is used to collect metrics from child tasks. It is empty when an - /// is instantiated (deserialized, created via [NetworkCoalesceExec::new_ready] etc...). - /// Metrics are populated in this map via [NetworkCoalesceExec::execute]. + /// metrics_collection is used to collect metrics from child tasks. It is initially + /// instantiated as an empty [DashMap] (see `try_decode` in `distributed_codec.rs`). + /// Metrics are populated here via [NetworkCoalesceExec::execute]. /// /// An instance may receive metrics for 0 to N child tasks, where N is the number of tasks in /// the stage it is reading from. This is because, by convention, the Worker sends metrics for @@ -84,7 +84,7 @@ pub struct NetworkCoalesceExec { impl NetworkCoalesceExec { /// Builds a new [NetworkCoalesceExec] in "Pending" state. /// - /// Typically, this node should be place right after nodes that coalesce all the input + /// Typically, this node should be placed right after nodes that coalesce all the input /// partitions into one, for example: /// - [CoalescePartitionsExec] /// - [SortPreservingMergeExec] @@ -97,7 +97,7 @@ impl NetworkCoalesceExec { ) -> Result { if task_count > 1 { return plan_err!( - "NetworkCoalesceExec cannot be executed in more than one task, {task_count} where passed." + "NetworkCoalesceExec cannot be executed in more than one task, {task_count} were passed." ); } Ok(Self { diff --git a/src/execution_plans/network_shuffle.rs b/src/execution_plans/network_shuffle.rs index 6c7a81ea..dcef4f47 100644 --- a/src/execution_plans/network_shuffle.rs +++ b/src/execution_plans/network_shuffle.rs @@ -40,10 +40,10 @@ use uuid::Uuid; /// /// The easiest way of thinking about this node is as a plan [RepartitionExec] node that is /// capable of fanning out the different produced partitions to different tasks. -/// This allows redistributing data across different tasks in different stages, that way different +/// This allows redistributing data across different tasks in different stages, so that different /// physical machines can make progress on different non-overlapping sets of data. /// -/// This node allows fanning out data from N tasks to M tasks, being N and M arbitrary non-zero +/// This node allows fanning out of data from N tasks to M tasks, with N and M being arbitrary non-zero /// positive numbers. Here are some examples of how data can be shuffled in different scenarios: /// /// # 1 to many @@ -108,12 +108,12 @@ use uuid::Uuid; /// /// The communication between two stages across a [NetworkShuffleExec] has two implications: /// -/// - Each task in Stage N+1 gather data from all tasks in Stage N -/// - The sum of the number of partitions in all tasks in Stage N+1 is equal to the +/// - Each task in Stage N+1 gathers data from all tasks in Stage N +/// - The total number of partitions across all tasks in Stage N+1 is equal to the /// number of partitions in a single task in Stage N. (e.g. (1,2,3,4)+(5,6,7,8) = (1,2,3,4,5,6,7,8) ) /// /// This node has two variants. -/// 1. Pending: it acts as a placeholder for the distributed optimization step to mark it as ready. +/// 1. Pending: acts as a placeholder for the distributed optimization step to mark it as ready. /// 2. Ready: runs within a distributed stage and queries the next input stage over the network /// using Arrow Flight. #[derive(Debug, Clone)] @@ -121,14 +121,14 @@ pub struct NetworkShuffleExec { /// the properties we advertise for this execution plan pub(crate) properties: PlanProperties, pub(crate) input_stage: Stage, - /// metrics_collection is used to collect metrics from child tasks. It is empty when an - /// is instantiated (deserialized, created via [NetworkShuffleExec::new_ready] etc...). - /// Metrics are populated in this map via [NetworkShuffleExec::execute]. + /// metrics_collection is used to collect metrics from child tasks. It is initially + /// instantiated as an empty [DashMap] (see `try_decode` in `distributed_codec.rs`). + /// Metrics are populated here via [NetworkCoalesceExec::execute]. /// /// An instance may receive metrics for 0 to N child tasks, where N is the number of tasks in - /// the stage it is reading from. This is because, by convention, the Worker - /// sends metrics for a task to the last NetworkShuffleExec to read from it, which may or may - /// not be this instance. + /// the stage it is reading from. This is because, by convention, the Worker sends metrics for + /// a task to the last NetworkCoalesceExec to read from it, which may or may not be this + /// instance. pub(crate) metrics_collection: Arc>>, } diff --git a/src/protobuf/distributed_codec.rs b/src/protobuf/distributed_codec.rs index c3b8351c..7d253313 100644 --- a/src/protobuf/distributed_codec.rs +++ b/src/protobuf/distributed_codec.rs @@ -82,7 +82,7 @@ impl PhysicalExtensionCodec for DistributedCodec { Ok(Stage { query_id: uuid::Uuid::from_slice(proto.query_id.as_ref()) - .map_err(|_| proto_error("Invalid query_id in ExecutionStageProto"))?, + .map_err(|_| proto_error("Invalid query_id in StageProto"))?, num: proto.num as usize, plan, tasks: decode_tasks(proto.tasks)?, diff --git a/src/stage.rs b/src/stage.rs index 5e80c219..dba7e56f 100644 --- a/src/stage.rs +++ b/src/stage.rs @@ -16,7 +16,7 @@ use uuid::Uuid; /// It implements [`ExecutionPlan`] and can be executed to produce a /// stream of record batches. /// -/// If the stage has input stages, then it those input stages will be executed on remote resources +/// If a stage has input stages, then those input stages will be executed on remote resources /// and will be provided the remainder of the stage tree. /// /// For example, if our stage tree looks like this: @@ -145,8 +145,8 @@ impl DistributedTaskContext { } impl Stage { - /// Creates a new `ExecutionStage` with the given plan and inputs. One task will be created - /// responsible for partitions in the plan. + /// Creates a new `Stage` with the given plan and inputs. `ExecutionTasks` will be created for + /// each of the `n_tasks` specified tasks. pub(crate) fn new( query_id: Uuid, num: usize, @@ -174,7 +174,7 @@ use prost::Message; /// /// The challenge to doing this at the moment is that `TreeRenderVisitor` /// in [`datafusion::physical_plan::display`] is not public, and that it also -/// is specific to a `ExecutionPlan` trait object, which we don't have. +/// is specific to an `ExecutionPlan` trait object, which we don't have. /// /// TODO: try to upstream a change to make rendering of Trees (logical, physical, stages) against /// a generic trait rather than a specific trait object. This would allow us to diff --git a/tests/tpch_correctness_test.rs b/tests/tpch_correctness_test.rs index 65e990ca..012dea32 100644 --- a/tests/tpch_correctness_test.rs +++ b/tests/tpch_correctness_test.rs @@ -132,6 +132,8 @@ mod tests { test_tpch_query(tpch::get_test_tpch_query(22)?).await } + // test_tpch_query runs each TPC-H query twice - once in a distributed manner and once + // in a non-distributed manner. For each query, it asserts that the results are identical. async fn test_tpch_query(sql: String) -> Result<(), Box> { let (ctx, _guard) = start_localhost_context(4, DefaultSessionBuilder).await; @@ -142,8 +144,6 @@ mod tests { Ok(()) } - // test_non_distributed_consistency runs each TPC-H query twice - once in a distributed manner - // and once in a non-distributed manner. For each query, it asserts that the results are identical. async fn run_tpch_query( ctx: SessionContext, sql: String, diff --git a/tests/tpch_explain_analyze.rs b/tests/tpch_explain_analyze.rs index f2175767..b03947f1 100644 --- a/tests/tpch_explain_analyze.rs +++ b/tests/tpch_explain_analyze.rs @@ -1069,14 +1069,14 @@ mod tests { Ok(()) } + // test_tpch_query runs each TPC-H query in a distributed manner while collecting metrics. + // This allows us to call `explain_analyze` on the resulting executed plan. async fn test_tpch_query(query_id: u8) -> Result> { let (mut ctx, _guard) = start_localhost_context(4, DefaultSessionBuilder).await; ctx.set_distributed_metrics_collection(true)?; run_tpch_query(ctx, query_id).await } - // test_non_distributed_consistency runs each TPC-H query twice - once in a distributed manner - // and once in a non-distributed manner. For each query, it asserts that the results are identical. async fn run_tpch_query(ctx: SessionContext, query_id: u8) -> Result> { let data_dir = ensure_tpch_data(TPCH_SCALE_FACTOR, TPCH_DATA_PARTS).await; let sql = tpch::get_test_tpch_query(query_id)?; diff --git a/tests/tpch_plans_test.rs b/tests/tpch_plans_test.rs index f7e89163..9a8a7a08 100644 --- a/tests/tpch_plans_test.rs +++ b/tests/tpch_plans_test.rs @@ -991,13 +991,12 @@ mod tests { Ok(()) } + // test_tpch_query generates and displays a distributed plan for each TPC-H query. async fn test_tpch_query(query_id: u8) -> Result> { let (ctx, _guard) = start_localhost_context(4, DefaultSessionBuilder).await; run_tpch_query(ctx, query_id).await } - // test_non_distributed_consistency runs each TPC-H query twice - once in a distributed manner - // and once in a non-distributed manner. For each query, it asserts that the results are identical. async fn run_tpch_query(ctx: SessionContext, query_id: u8) -> Result> { let data_dir = ensure_tpch_data(TPCH_SCALE_FACTOR, TPCH_DATA_PARTS).await; let sql = tpch::get_test_tpch_query(query_id)?; From 057b9e59ef9de39b9c4b09eae3932f6ffeb9632f Mon Sep 17 00:00:00 2001 From: Edson Petry <124717297+EdsonPetry@users.noreply.github.com> Date: Tue, 13 Jan 2026 07:07:55 -0500 Subject: [PATCH 17/27] add binary executable crate for observability tool (#290) * feat: add binary crate for ddf observability tool * Revert "feat: add binary crate for ddf observability tool" This reverts commit e6200a1d5daeefc7e404f187e9a816f69659adf3. * feat: add binary crate for DDF observability and monitoring tool * feat: add ratatui --- Cargo.lock | 1004 ++++++++++++++++++++++++++++++++++++++++--- Cargo.toml | 2 +- console/Cargo.toml | 9 + console/src/main.rs | 20 + 4 files changed, 975 insertions(+), 60 deletions(-) create mode 100644 console/Cargo.toml create mode 100644 console/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 64242eed..3235fa82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.1" @@ -160,7 +169,7 @@ dependencies = [ "snap", "strum 0.27.2", "strum_macros 0.27.2", - "thiserror", + "thiserror 2.0.17", "uuid", "xz2", "zstd", @@ -172,7 +181,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0c269894b6fe5e9d7ada0cf69b5bf847ff35bc25fc271f08e1d080fce80339a" dependencies = [ - "object", + "object 0.32.2", ] [[package]] @@ -719,6 +728,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89cbf775b137e9b968e67227ef7f775587cde3fd31b0d8599dbd0f598a48340" +dependencies = [ + "bytemuck", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -1259,6 +1277,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object 0.37.3", + "rustc-demangle", + "windows-link", +] + [[package]] name = "ballista" version = "50.0.0" @@ -1291,7 +1324,7 @@ dependencies = [ "datafusion-proto 50.3.0", "datafusion-proto-common 50.3.0", "futures", - "itertools", + "itertools 0.14.0", "log", "md-5", "object_store", @@ -1401,6 +1434,21 @@ dependencies = [ "serde", ] +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "1.3.2" @@ -1460,7 +1508,7 @@ version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77e9d642a7e3a318e37c2c9427b5a6a48aa1ad55dcd986f3034ab2239045a645" dependencies = [ - "darling", + "darling 0.21.3", "ident_case", "prettyplease", "proc-macro2", @@ -1496,6 +1544,12 @@ version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +[[package]] +name = "bytemuck" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" + [[package]] name = "byteorder" version = "1.5.0" @@ -1546,6 +1600,15 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.47" @@ -1591,7 +1654,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" dependencies = [ "chrono", - "phf", + "phf 0.12.1", ] [[package]] @@ -1677,6 +1740,33 @@ dependencies = [ "cc", ] +[[package]] +name = "color-eyre" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5920befb47832a6d61ee3a3a846565cfa39b331331e68a3b1d1116630f2f26d" +dependencies = [ + "backtrace", + "color-spantrace", + "eyre", + "indenter", + "once_cell", + "owo-colors", + "tracing-error", +] + +[[package]] +name = "color-spantrace" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8b88ea9df13354b55bc7234ebcce36e6ef896aca2e42a15de9e10edce01b427" +dependencies = [ + "once_cell", + "owo-colors", + "tracing-core", + "tracing-error", +] + [[package]] name = "colorchoice" version = "1.0.4" @@ -1694,6 +1784,29 @@ dependencies = [ "unicode-width 0.2.2", ] +[[package]] +name = "compact_str" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "console" +version = "0.1.0" +dependencies = [ + "color-eyre", + "crossterm", + "ratatui", +] + [[package]] name = "console" version = "0.15.11" @@ -1732,6 +1845,15 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -1815,6 +1937,33 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crossterm" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" +dependencies = [ + "bitflags 2.10.0", + "crossterm_winapi", + "derive_more", + "document-features", + "mio", + "parking_lot", + "rustix", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + [[package]] name = "crunchy" version = "0.2.4" @@ -1831,6 +1980,16 @@ dependencies = [ "typenum", ] +[[package]] +name = "csscolorparser" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2a7d3066da2de787b7f032c736763eb7ae5d355f81a68bab2675a96008b0bf" +dependencies = [ + "lab", + "phf 0.11.3", +] + [[package]] name = "csv" version = "1.4.0" @@ -1858,8 +2017,18 @@ version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.21.3", + "darling_macro 0.21.3", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core 0.23.0", + "darling_macro 0.23.0", ] [[package]] @@ -1876,13 +2045,37 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim 0.11.1", + "syn 2.0.110", +] + [[package]] name = "darling_macro" version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ - "darling_core", + "darling_core 0.21.3", + "quote", + "syn 2.0.110", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core 0.23.0", "quote", "syn 2.0.110", ] @@ -1940,7 +2133,7 @@ dependencies = [ "datafusion-sql 50.3.0", "flate2", "futures", - "itertools", + "itertools 0.14.0", "log", "object_store", "parking_lot", @@ -1996,7 +2189,7 @@ dependencies = [ "datafusion-sql 51.0.0", "flate2", "futures", - "itertools", + "itertools 0.14.0", "log", "object_store", "parking_lot", @@ -2032,7 +2225,7 @@ dependencies = [ "datafusion-session 50.3.0", "datafusion-sql 50.3.0", "futures", - "itertools", + "itertools 0.14.0", "log", "object_store", "parking_lot", @@ -2057,7 +2250,7 @@ dependencies = [ "datafusion-physical-plan 51.0.0", "datafusion-session 51.0.0", "futures", - "itertools", + "itertools 0.14.0", "log", "object_store", "parking_lot", @@ -2105,7 +2298,7 @@ dependencies = [ "datafusion-physical-expr-common 51.0.0", "datafusion-physical-plan 51.0.0", "futures", - "itertools", + "itertools 0.14.0", "log", "object_store", "tokio", @@ -2236,7 +2429,7 @@ dependencies = [ "flate2", "futures", "glob", - "itertools", + "itertools 0.14.0", "log", "object_store", "parquet 56.2.0", @@ -2273,7 +2466,7 @@ dependencies = [ "flate2", "futures", "glob", - "itertools", + "itertools 0.14.0", "log", "object_store", "rand 0.9.2", @@ -2303,7 +2496,7 @@ dependencies = [ "datafusion-physical-plan 51.0.0", "datafusion-session 51.0.0", "futures", - "itertools", + "itertools 0.14.0", "object_store", "tokio", ] @@ -2447,7 +2640,7 @@ dependencies = [ "datafusion-pruning 50.3.0", "datafusion-session 50.3.0", "futures", - "itertools", + "itertools 0.14.0", "log", "object_store", "parking_lot", @@ -2478,7 +2671,7 @@ dependencies = [ "datafusion-pruning 51.0.0", "datafusion-session 51.0.0", "futures", - "itertools", + "itertools 0.14.0", "log", "object_store", "parking_lot", @@ -2504,7 +2697,7 @@ dependencies = [ "http 1.3.1", "hyper-util", "insta", - "itertools", + "itertools 0.14.0", "moka", "object_store", "parquet 57.1.0", @@ -2670,7 +2863,7 @@ dependencies = [ "datafusion-functions-window-common 51.0.0", "datafusion-physical-expr-common 51.0.0", "indexmap", - "itertools", + "itertools 0.14.0", "paste", "recursive", "serde_json", @@ -2686,7 +2879,7 @@ dependencies = [ "arrow 56.2.0", "datafusion-common 50.3.0", "indexmap", - "itertools", + "itertools 0.14.0", "paste", ] @@ -2699,7 +2892,7 @@ dependencies = [ "arrow 57.1.0", "datafusion-common 51.0.0", "indexmap", - "itertools", + "itertools 0.14.0", "paste", ] @@ -2722,7 +2915,7 @@ dependencies = [ "datafusion-expr-common 50.3.0", "datafusion-macros 50.3.0", "hex", - "itertools", + "itertools 0.14.0", "log", "md-5", "rand 0.9.2", @@ -2751,7 +2944,7 @@ dependencies = [ "datafusion-expr-common 51.0.0", "datafusion-macros 51.0.0", "hex", - "itertools", + "itertools 0.14.0", "log", "md-5", "num-traits", @@ -2847,7 +3040,7 @@ dependencies = [ "datafusion-functions-aggregate-common 50.3.0", "datafusion-macros 50.3.0", "datafusion-physical-expr-common 50.3.0", - "itertools", + "itertools 0.14.0", "log", "paste", ] @@ -2870,7 +3063,7 @@ dependencies = [ "datafusion-functions-aggregate-common 51.0.0", "datafusion-macros 51.0.0", "datafusion-physical-expr-common 51.0.0", - "itertools", + "itertools 0.14.0", "log", "paste", ] @@ -2998,7 +3191,7 @@ dependencies = [ "datafusion-expr-common 50.3.0", "datafusion-physical-expr 50.3.0", "indexmap", - "itertools", + "itertools 0.14.0", "log", "recursive", "regex", @@ -3018,7 +3211,7 @@ dependencies = [ "datafusion-expr-common 51.0.0", "datafusion-physical-expr 51.0.0", "indexmap", - "itertools", + "itertools 0.14.0", "log", "recursive", "regex", @@ -3041,7 +3234,7 @@ dependencies = [ "half", "hashbrown 0.14.5", "indexmap", - "itertools", + "itertools 0.14.0", "log", "parking_lot", "paste", @@ -3064,7 +3257,7 @@ dependencies = [ "half", "hashbrown 0.14.5", "indexmap", - "itertools", + "itertools 0.14.0", "parking_lot", "paste", "petgraph 0.8.3", @@ -3082,7 +3275,7 @@ dependencies = [ "datafusion-functions 50.3.0", "datafusion-physical-expr 50.3.0", "datafusion-physical-expr-common 50.3.0", - "itertools", + "itertools 0.14.0", ] [[package]] @@ -3097,7 +3290,7 @@ dependencies = [ "datafusion-functions 51.0.0", "datafusion-physical-expr 51.0.0", "datafusion-physical-expr-common 51.0.0", - "itertools", + "itertools 0.14.0", ] [[package]] @@ -3111,7 +3304,7 @@ dependencies = [ "datafusion-common 50.3.0", "datafusion-expr-common 50.3.0", "hashbrown 0.14.5", - "itertools", + "itertools 0.14.0", ] [[package]] @@ -3125,7 +3318,7 @@ dependencies = [ "datafusion-common 51.0.0", "datafusion-expr-common 51.0.0", "hashbrown 0.14.5", - "itertools", + "itertools 0.14.0", ] [[package]] @@ -3143,7 +3336,7 @@ dependencies = [ "datafusion-physical-expr-common 50.3.0", "datafusion-physical-plan 50.3.0", "datafusion-pruning 50.3.0", - "itertools", + "itertools 0.14.0", "log", "recursive", ] @@ -3163,7 +3356,7 @@ dependencies = [ "datafusion-physical-expr-common 51.0.0", "datafusion-physical-plan 51.0.0", "datafusion-pruning 51.0.0", - "itertools", + "itertools 0.14.0", "recursive", ] @@ -3191,7 +3384,7 @@ dependencies = [ "half", "hashbrown 0.14.5", "indexmap", - "itertools", + "itertools 0.14.0", "log", "parking_lot", "pin-project-lite", @@ -3222,7 +3415,7 @@ dependencies = [ "half", "hashbrown 0.14.5", "indexmap", - "itertools", + "itertools 0.14.0", "log", "parking_lot", "pin-project-lite", @@ -3308,7 +3501,7 @@ dependencies = [ "datafusion-physical-expr 50.3.0", "datafusion-physical-expr-common 50.3.0", "datafusion-physical-plan 50.3.0", - "itertools", + "itertools 0.14.0", "log", ] @@ -3325,7 +3518,7 @@ dependencies = [ "datafusion-physical-expr 51.0.0", "datafusion-physical-expr-common 51.0.0", "datafusion-physical-plan 51.0.0", - "itertools", + "itertools 0.14.0", "log", ] @@ -3346,7 +3539,7 @@ dependencies = [ "datafusion-physical-plan 50.3.0", "datafusion-sql 50.3.0", "futures", - "itertools", + "itertools 0.14.0", "log", "object_store", "parking_lot", @@ -3419,6 +3612,12 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "deltae" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5729f5117e208430e437df2f4843f5e5952997175992d1414f94c57d61e270b4" + [[package]] name = "deranged" version = "0.5.5" @@ -3439,6 +3638,28 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.110", +] + [[package]] name = "diff" version = "0.1.13" @@ -3488,6 +3709,15 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + [[package]] name = "dunce" version = "1.0.5" @@ -3566,6 +3796,35 @@ version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" +[[package]] +name = "euclid" +version = "0.22.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad9cdb4b747e485a12abb0e6566612956c7a1bafa3bdb8d682c5b6d403589e48" +dependencies = [ + "num-traits", +] + +[[package]] +name = "eyre" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +dependencies = [ + "indenter", + "once_cell", +] + +[[package]] +name = "fancy-regex" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b95f7c0680e4142284cf8b22c14a476e87d61b004a3a0861872b32ef7ead40a2" +dependencies = [ + "bit-set", + "regex", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -3583,12 +3842,35 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "filedescriptor" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e40758ed24c9b2eeb76c35fb0aebc66c626084edd827e07e1552279814c6682d" +dependencies = [ + "libc", + "thiserror 1.0.69", + "winapi", +] + [[package]] name = "find-msvc-tools" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" +[[package]] +name = "finl_unicode" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9844ddc3a6e533d62bba727eb6c28b5d360921d5175e9ff0f1e621a5c590a4d5" + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "fixedbitset" version = "0.5.7" @@ -3628,6 +3910,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -3790,6 +4078,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + [[package]] name = "glob" version = "0.3.3" @@ -3862,7 +4156,7 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -3870,6 +4164,11 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "heck" @@ -4259,6 +4558,12 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indenter" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5" + [[package]] name = "indexmap" version = "2.12.1" @@ -4269,6 +4574,15 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + [[package]] name = "inout" version = "0.1.4" @@ -4284,12 +4598,25 @@ version = "1.44.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8732d3774162a0851e3f2b150eb98f31a9885dd75985099421d393385a01dfd" dependencies = [ - "console", + "console 0.15.11", "once_cell", "regex", "similar", ] +[[package]] +name = "instability" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357b7205c6cd18dd2c86ed312d1e70add149aea98e7ef72b9fdf0270e555c11d" +dependencies = [ + "darling 0.23.0", + "indoc", + "proc-macro2", + "quote", + "syn 2.0.110", +] + [[package]] name = "integer-encoding" version = "3.0.4" @@ -4318,6 +4645,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -4377,6 +4713,23 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kasuari" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fe90c1150662e858c7d5f945089b7517b0a80d8bf7ba4b1b5ffc984e7230a5b" +dependencies = [ + "hashbrown 0.16.1", + "portable-atomic", + "thiserror 2.0.17", +] + +[[package]] +name = "lab" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf36173d4167ed999940f804952e6b08197cae5ad5d572eb4db150ce8ad5d58f" + [[package]] name = "lazy_static" version = "1.5.0" @@ -4487,6 +4840,15 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "line-clipping" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f4de44e98ddbf09375cbf4d17714d18f39195f4f4894e8524501726fd9a8a4a" +dependencies = [ + "bitflags 2.10.0", +] + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -4499,6 +4861,12 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + [[package]] name = "lock_api" version = "0.4.14" @@ -4514,6 +4882,15 @@ version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +[[package]] +name = "lru" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -4559,6 +4936,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "mac_address" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0aeb26bf5e836cc1c341c8106051b573f1766dfa05aa87f0b98be5e51b02303" +dependencies = [ + "nix 0.29.0", + "winapi", +] + [[package]] name = "matchers" version = "0.2.0" @@ -4597,7 +4984,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] -name = "mimalloc" +name = "memmem" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a64a92489e2744ce060c349162be1c5f33c6969234104dbd99ddb5feb08b8c15" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mimalloc" version = "0.1.48" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1ee66a4b64c74f4ef288bcbb9192ad9c3feaad75193129ac8509af543894fd8" @@ -4611,6 +5013,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -4628,6 +5036,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" dependencies = [ "libc", + "log", "wasi", "windows-sys 0.61.2", ] @@ -4681,6 +5090,19 @@ dependencies = [ "smallvec", ] +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.10.0", + "cfg-if", + "cfg_aliases", + "libc", + "memoffset", +] + [[package]] name = "nix" version = "0.30.1" @@ -4693,6 +5115,16 @@ dependencies = [ "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -4742,6 +5174,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.110", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -4783,6 +5226,15 @@ dependencies = [ "libm", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + [[package]] name = "object" version = "0.32.2" @@ -4792,6 +5244,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "object_store" version = "0.12.4" @@ -4808,7 +5269,7 @@ dependencies = [ "http-body-util", "humantime", "hyper 1.8.1", - "itertools", + "itertools 0.14.0", "md-5", "parking_lot", "percent-encoding", @@ -4820,7 +5281,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "thiserror", + "thiserror 2.0.17", "tokio", "tracing", "url", @@ -4910,12 +5371,27 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + [[package]] name = "outref" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +[[package]] +name = "owo-colors" +version = "4.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c6901729fa79e91a0913333229e9ca5dc725089d1c363b2f4b4760709dc4a52" + [[package]] name = "parking_lot" version = "0.12.5" @@ -5036,13 +5512,56 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pest" +version = "2.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9eb05c21a464ea704b53158d358a31e6425db2f63a1a7312268b05fe2b75f7" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f9dbced329c441fa79d80472764b1a2c7e57123553b8519b36663a2fb234ed" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bb96d5051a78f44f43c8f712d8e810adb0ebf923fc9ed2655a7f66f63ba8ee5" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.110", +] + +[[package]] +name = "pest_meta" +version = "2.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "602113b5b5e8621770cfd490cfd90b9f84ab29bd2b0e49ad83eb6d186cef2365" +dependencies = [ + "pest", + "sha2", +] + [[package]] name = "petgraph" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ - "fixedbitset", + "fixedbitset 0.5.7", "indexmap", ] @@ -5052,19 +5571,71 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ - "fixedbitset", + "fixedbitset 0.5.7", "hashbrown 0.15.5", "indexmap", "serde", ] +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared 0.11.3", +] + [[package]] name = "phf" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" dependencies = [ - "phf_shared", + "phf_shared 0.12.1", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared 0.11.3", + "rand 0.8.5", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared 0.11.3", + "proc-macro2", + "quote", + "syn 2.0.110", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", ] [[package]] @@ -5248,7 +5819,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -5268,7 +5839,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.110", @@ -5281,7 +5852,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" dependencies = [ "anyhow", - "itertools", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.110", @@ -5345,7 +5916,7 @@ dependencies = [ "rustc-hash", "rustls 0.23.35", "socket2 0.6.1", - "thiserror", + "thiserror 2.0.17", "tokio", "tracing", "web-time", @@ -5366,7 +5937,7 @@ dependencies = [ "rustls 0.23.35", "rustls-pki-types", "slab", - "thiserror", + "thiserror 2.0.17", "tinyvec", "tracing", "web-time", @@ -5470,6 +6041,91 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "ratatui" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1ce67fb8ba4446454d1c8dbaeda0557ff5e94d39d5e5ed7f10a65eb4c8266bc" +dependencies = [ + "instability", + "ratatui-core", + "ratatui-crossterm", + "ratatui-macros", + "ratatui-termwiz", + "ratatui-widgets", +] + +[[package]] +name = "ratatui-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ef8dea09a92caaf73bff7adb70b76162e5937524058a7e5bff37869cbbec293" +dependencies = [ + "bitflags 2.10.0", + "compact_str", + "hashbrown 0.16.1", + "indoc", + "itertools 0.14.0", + "kasuari", + "lru", + "strum 0.27.2", + "thiserror 2.0.17", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.2.2", +] + +[[package]] +name = "ratatui-crossterm" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c9b9f652b4c121fb25c6a391dd06406d3b092ba68827e6d2f09550edc54b3" +dependencies = [ + "cfg-if", + "crossterm", + "instability", + "ratatui-core", +] + +[[package]] +name = "ratatui-macros" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7f1342a13e83e4bb9d0b793d0ea762be633f9582048c892ae9041ef39c936f4" +dependencies = [ + "ratatui-core", + "ratatui-widgets", +] + +[[package]] +name = "ratatui-termwiz" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f76fe0bd0ed4295f0321b1676732e2454024c15a35d01904ddb315afd3d545c" +dependencies = [ + "ratatui-core", + "termwiz", +] + +[[package]] +name = "ratatui-widgets" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7dbfa023cd4e604c2553483820c5fe8aa9d71a42eea5aa77c6e7f35756612db" +dependencies = [ + "bitflags 2.10.0", + "hashbrown 0.16.1", + "indoc", + "instability", + "itertools 0.14.0", + "line-clipping", + "ratatui-core", + "strum 0.27.2", + "time", + "unicode-segmentation", + "unicode-width 0.2.2", +] + [[package]] name = "recursive" version = "0.1.1" @@ -5507,7 +6163,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.16", "libredox", - "thiserror", + "thiserror 2.0.17", ] [[package]] @@ -5641,6 +6297,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "rustc-demangle" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" + [[package]] name = "rustc-hash" version = "2.1.1" @@ -5769,7 +6431,7 @@ dependencies = [ "libc", "log", "memchr", - "nix", + "nix 0.30.1", "radix_trie", "unicode-segmentation", "unicode-width 0.2.2", @@ -5978,6 +6640,27 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + [[package]] name = "signal-hook-registry" version = "1.4.6" @@ -6101,6 +6784,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strsim" version = "0.8.0" @@ -6148,6 +6837,9 @@ name = "strum" version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros 0.27.2", +] [[package]] name = "strum_macros" @@ -6262,6 +6954,69 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "terminfo" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ea810f0692f9f51b382fff5893887bb4580f5fa246fde546e0b13e7fcee662" +dependencies = [ + "fnv", + "nom", + "phf 0.11.3", + "phf_codegen", +] + +[[package]] +name = "termios" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411c5bf740737c7918b8b1fe232dca4dc9f8e754b8ad5e20966814001ed0ac6b" +dependencies = [ + "libc", +] + +[[package]] +name = "termwiz" +version = "0.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4676b37242ccbd1aabf56edb093a4827dc49086c0ffd764a5705899e0f35f8f7" +dependencies = [ + "anyhow", + "base64", + "bitflags 2.10.0", + "fancy-regex", + "filedescriptor", + "finl_unicode", + "fixedbitset 0.4.2", + "hex", + "lazy_static", + "libc", + "log", + "memmem", + "nix 0.29.0", + "num-derive", + "num-traits", + "ordered-float 4.6.0", + "pest", + "pest_derive", + "phf 0.11.3", + "sha2", + "signal-hook", + "siphasher", + "terminfo", + "termios", + "thiserror 1.0.69", + "ucd-trie", + "unicode-segmentation", + "vtparse", + "wezterm-bidi", + "wezterm-blob-leases", + "wezterm-color-types", + "wezterm-dynamic", + "wezterm-input-types", + "winapi", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -6271,13 +7026,33 @@ dependencies = [ "unicode-width 0.1.14", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.17", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.110", ] [[package]] @@ -6308,7 +7083,7 @@ checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" dependencies = [ "byteorder", "integer-encoding", - "ordered-float", + "ordered-float 2.10.1", ] [[package]] @@ -6319,7 +7094,9 @@ checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" dependencies = [ "deranged", "itoa", + "libc", "num-conv", + "num_threads", "powerfmt", "serde", "time-core", @@ -6653,7 +7430,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf" dependencies = [ "crossbeam-channel", - "thiserror", + "thiserror 2.0.17", "time", "tracing-subscriber", ] @@ -6679,6 +7456,16 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-error" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db" +dependencies = [ + "tracing", + "tracing-subscriber", +] + [[package]] name = "tracing-log" version = "0.2.0" @@ -6726,6 +7513,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + [[package]] name = "unicode-ident" version = "1.0.22" @@ -6738,6 +7531,17 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +[[package]] +name = "unicode-truncate" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fbf03860ff438702f3910ca5f28f8dac63c1c11e7efb5012b8b175493606330" +dependencies = [ + "itertools 0.13.0", + "unicode-segmentation", + "unicode-width 0.2.2", +] + [[package]] name = "unicode-width" version = "0.1.14" @@ -6792,6 +7596,7 @@ version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" dependencies = [ + "atomic", "getrandom 0.3.4", "js-sys", "serde", @@ -6828,6 +7633,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +[[package]] +name = "vtparse" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9b2acfb050df409c972a37d3b8e08cdea3bddb0c09db9d53137e504cfabed0" +dependencies = [ + "utf8parse", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -6953,6 +7767,78 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "wezterm-bidi" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0a6e355560527dd2d1cf7890652f4f09bb3433b6aadade4c9b5ed76de5f3ec" +dependencies = [ + "log", + "wezterm-dynamic", +] + +[[package]] +name = "wezterm-blob-leases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692daff6d93d94e29e4114544ef6d5c942a7ed998b37abdc19b17136ea428eb7" +dependencies = [ + "getrandom 0.3.4", + "mac_address", + "sha2", + "thiserror 1.0.69", + "uuid", +] + +[[package]] +name = "wezterm-color-types" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7de81ef35c9010270d63772bebef2f2d6d1f2d20a983d27505ac850b8c4b4296" +dependencies = [ + "csscolorparser", + "deltae", + "lazy_static", + "wezterm-dynamic", +] + +[[package]] +name = "wezterm-dynamic" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f2ab60e120fd6eaa68d9567f3226e876684639d22a4219b313ff69ec0ccd5ac" +dependencies = [ + "log", + "ordered-float 4.6.0", + "strsim 0.11.1", + "thiserror 1.0.69", + "wezterm-dynamic-derive", +] + +[[package]] +name = "wezterm-dynamic-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c0cf2d539c645b448eaffec9ec494b8b19bd5077d9e58cb1ae7efece8d575b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "wezterm-input-types" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7012add459f951456ec9d6c7e6fc340b1ce15d6fc9629f8c42853412c029e57e" +dependencies = [ + "bitflags 1.3.2", + "euclid", + "lazy_static", + "serde", + "wezterm-dynamic", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 4770e155..33c77388 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["benchmarks", "cli"] +members = ["benchmarks", "cli", "console"] [workspace.dependencies] datafusion = { version = "51.0.0", default-features = false } diff --git a/console/Cargo.toml b/console/Cargo.toml new file mode 100644 index 00000000..b01722dd --- /dev/null +++ b/console/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "console" +version = "0.1.0" +edition = "2024" + +[dependencies] +color-eyre = "0.6.5" +crossterm = "0.29.0" +ratatui = "0.30.0" diff --git a/console/src/main.rs b/console/src/main.rs new file mode 100644 index 00000000..a686bef9 --- /dev/null +++ b/console/src/main.rs @@ -0,0 +1,20 @@ +use ratatui::{DefaultTerminal, Frame}; + +fn main() -> color_eyre::Result<()> { + color_eyre::install()?; + ratatui::run(app)?; + Ok(()) +} + +fn app(terminal: &mut DefaultTerminal) -> std::io::Result<()> { + loop { + terminal.draw(render)?; + if crossterm::event::read()?.is_key_press() { + break Ok(()); + } + } +} + +fn render(frame: &mut Frame) { + frame.render_widget("hello world", frame.area()); +} From 73de13453f8024845c9c5e64f969126ff6449c27 Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Tue, 13 Jan 2026 18:09:17 +0100 Subject: [PATCH 18/27] Identify benchmark queries as arbitrary strings (#282) * Allow passing any query id for benchmarks * Allow passing a custom query * Tidy up benchmarking code * Respond to PR feedbac --- benchmarks/cdk/bin/@bench-common.ts | 20 ++- benchmarks/cdk/bin/ballista-bench.ts | 4 +- benchmarks/cdk/bin/datafusion-bench.ts | 4 +- benchmarks/cdk/bin/spark-bench.ts | 4 +- benchmarks/cdk/bin/trino-bench.ts | 4 +- benchmarks/src/prepare_tpcds.rs | 2 +- benchmarks/src/prepare_tpch.rs | 105 ++++++++++++- benchmarks/src/run.rs | 24 +-- src/test_utils/benchmarks_common.rs | 62 ++++++++ src/test_utils/clickbench.rs | 52 +----- src/test_utils/mod.rs | 1 + src/test_utils/tpcds.rs | 55 ++----- src/test_utils/tpch.rs | 124 +-------------- tests/clickbench_correctness_test.rs | 96 +++++------ tests/clickbench_plans_test.rs | 94 +++++------ tests/tpcds_correctness_test.rs | 210 ++++++++++++------------- tests/tpcds_plans_test.rs | 208 ++++++++++++------------ tests/tpch_correctness_test.rs | 59 +++---- tests/tpch_explain_analyze.rs | 68 ++++---- tests/tpch_plans_test.rs | 69 ++++---- 20 files changed, 599 insertions(+), 666 deletions(-) create mode 100644 src/test_utils/benchmarks_common.rs diff --git a/benchmarks/cdk/bin/@bench-common.ts b/benchmarks/cdk/bin/@bench-common.ts index a37fea62..ed9e3767 100644 --- a/benchmarks/cdk/bin/@bench-common.ts +++ b/benchmarks/cdk/bin/@bench-common.ts @@ -111,16 +111,16 @@ async function isDirWithAllParquetFiles(dir: string): Promise { return true } -async function queriesForDataset(dataset: string): Promise<[string, string][]> { +async function queriesForDataset(dataset: string): Promise<{ id: string, sql: string }[]> { const datasetSuffix = dataset.split("_")[0] const queriesPath = path.join(ROOT, "testdata", datasetSuffix, "queries") - const queries: [string, string][] = [] - for (const queryName of await fs.readdir(queriesPath)) { - const sql = await fs.readFile(path.join(queriesPath, queryName), 'utf-8'); - queries.push([queryName, sql]) + const queries = [] + for (const fileName of await fs.readdir(queriesPath)) { + const sql = await fs.readFile(path.join(queriesPath, fileName), 'utf-8'); + queries.push({ id: fileName.replace(".sql", ""), sql }) } - queries.sort(([name1], [name2]) => numericId(name1) > numericId(name2) ? 1 : -1) + queries.sort((a, b) => numericId(a.id) > numericId(b.id) ? 1 : -1) return queries } @@ -133,7 +133,7 @@ export async function runBenchmark( options: { dataset: string iterations: number; - queries: number[]; + queries: string[]; outputPath: string; } ) { @@ -145,15 +145,13 @@ export async function runBenchmark( const s3Paths = await tablePathsForDataset(dataset) await runner.createTables(s3Paths); - for (const [queryName, sql] of await queriesForDataset(dataset)) { - const id = numericId(queryName) - + for (const { id, sql } of await queriesForDataset(dataset)) { if (queries.length > 0 && !queries.includes(id)) { continue; } const queryResult: QueryResult = { - query: queryName, + query: id, iterations: [], }; diff --git a/benchmarks/cdk/bin/ballista-bench.ts b/benchmarks/cdk/bin/ballista-bench.ts index 02b9d440..a2755f37 100644 --- a/benchmarks/cdk/bin/ballista-bench.ts +++ b/benchmarks/cdk/bin/ballista-bench.ts @@ -12,14 +12,14 @@ async function main() { program .option('--dataset ', 'Dataset to run queries on') .option('-i, --iterations ', 'Number of iterations', '3') - .option('--query ', 'A specific query to run', undefined) + .option('--queries ', 'Specific queries to run', undefined) .parse(process.argv); const options = program.opts(); const dataset: string = options.dataset const iterations = parseInt(options.iterations); - const queries = options.query ? [parseInt(options.query)] : []; + const queries = options.queries?.split(",") ?? [] const runner = new BallistaRunner({}); diff --git a/benchmarks/cdk/bin/datafusion-bench.ts b/benchmarks/cdk/bin/datafusion-bench.ts index c817a71c..a3f84ea3 100644 --- a/benchmarks/cdk/bin/datafusion-bench.ts +++ b/benchmarks/cdk/bin/datafusion-bench.ts @@ -16,7 +16,7 @@ async function main() { .option('--cardinality-task-sf ', 'Cardinality task scale factor', '2') .option('--shuffle-batch-size ', 'Shuffle batch coalescing size (number of rows)', '8192') .option('--collect-metrics ', 'Propagates metric collection', 'true') - .option('--query ', 'A specific query to run', undefined) + .option('--queries ', 'Specific queries to run', undefined) .parse(process.argv); const options = program.opts(); @@ -26,7 +26,7 @@ async function main() { const filesPerTask = parseInt(options.filesPerTask); const cardinalityTaskSf = parseInt(options.cardinalityTaskSf); const shuffleBatchSize = parseInt(options.shuffleBatchSize); - const queries = options.query ? [parseInt(options.query)] : []; + const queries = options.queries?.split(",") ?? [] const collectMetrics = options.collectMetrics === 'true' || options.collectMetrics === 1 const runner = new DataFusionRunner({ diff --git a/benchmarks/cdk/bin/spark-bench.ts b/benchmarks/cdk/bin/spark-bench.ts index 2588e0df..1b8ef4ce 100644 --- a/benchmarks/cdk/bin/spark-bench.ts +++ b/benchmarks/cdk/bin/spark-bench.ts @@ -12,14 +12,14 @@ async function main() { program .option('--dataset ', 'Dataset to run queries on') .option('-i, --iterations ', 'Number of iterations', '3') - .option('--query ', 'A specific query to run', undefined) + .option('--queries ', 'Specific queries to run', undefined) .parse(process.argv); const options = program.opts(); const dataset: string = options.dataset const iterations = parseInt(options.iterations); - const queries = options.query ? [parseInt(options.query)] : []; + const queries = options.queries?.split(",") ?? [] const runner = new SparkRunner({}); diff --git a/benchmarks/cdk/bin/trino-bench.ts b/benchmarks/cdk/bin/trino-bench.ts index 136b3a0f..abf213b9 100644 --- a/benchmarks/cdk/bin/trino-bench.ts +++ b/benchmarks/cdk/bin/trino-bench.ts @@ -11,14 +11,14 @@ async function main() { program .option('--dataset ', 'Scale factor', '1') .option('-i, --iterations ', 'Number of iterations', '3') - .option('--query ', 'A specific query to run', undefined) + .option('--queries ', 'Specific queries to run', undefined) .parse(process.argv); const options = program.opts(); const dataset: string = options.dataset const iterations = parseInt(options.iterations); - const queries = options.query ? [parseInt(options.query)] : []; + const queries = options.queries?.split(",") ?? [] const datasetPath = path.join(ROOT, "benchmarks", "data", dataset); const outputPath = path.join(datasetPath, "remote-results.json") diff --git a/benchmarks/src/prepare_tpcds.rs b/benchmarks/src/prepare_tpcds.rs index b6c10348..a1ab1b47 100644 --- a/benchmarks/src/prepare_tpcds.rs +++ b/benchmarks/src/prepare_tpcds.rs @@ -21,7 +21,7 @@ pub struct PrepareTpcdsOpt { impl PrepareTpcdsOpt { pub async fn run(self) -> datafusion::common::Result<()> { - tpcds::generate_tpcds_data(Path::new(&self.output_path), self.sf, self.partitions) + tpcds::generate_data(Path::new(&self.output_path), self.sf, self.partitions) .await .map_err(|e| DataFusionError::Internal(format!("{e:?}"))) } diff --git a/benchmarks/src/prepare_tpch.rs b/benchmarks/src/prepare_tpch.rs index 826ed56a..b6e07499 100644 --- a/benchmarks/src/prepare_tpch.rs +++ b/benchmarks/src/prepare_tpch.rs @@ -15,13 +15,13 @@ // specific language governing permissions and limitations // under the License. +use datafusion::arrow::datatypes::{DataType, Field}; use datafusion::common::instant::Instant; use datafusion::error::Result; use datafusion::logical_expr::select_expr::SelectExpr; use datafusion::parquet::basic::Compression; use datafusion::parquet::file::properties::WriterProperties; use datafusion::prelude::*; -use datafusion_distributed::test_utils::tpch; use std::fs; use std::path::{Path, PathBuf}; use structopt::StructOpt; @@ -52,9 +52,9 @@ impl PrepareTpchOpt { let output_path = self.output_path.to_str().unwrap(); let output_root_path = Path::new(output_path); - for table in tpch::TPCH_TABLES { + for table in TPCH_TABLES { let start = Instant::now(); - let schema = tpch::get_tpch_table_schema(table); + let schema = get_tpch_table_schema(table); let key_column_name = schema.fields()[0].name(); let input_path = format!("{input_path}/{table}.tbl"); @@ -113,3 +113,102 @@ impl PrepareTpchOpt { Ok(()) } } + +const TPCH_TABLES: &[&str] = &[ + "part", "supplier", "partsupp", "customer", "orders", "lineitem", "nation", "region", +]; + +pub fn get_tpch_table_schema(table: &str) -> datafusion::arrow::datatypes::Schema { + // note that the schema intentionally uses signed integers so that any generated Parquet + // files can also be used to benchmark tools that only support signed integers, such as + // Apache Spark + + match table { + "part" => datafusion::arrow::datatypes::Schema::new(vec![ + Field::new("p_partkey", DataType::Int64, false), + Field::new("p_name", DataType::Utf8, false), + Field::new("p_mfgr", DataType::Utf8, false), + Field::new("p_brand", DataType::Utf8, false), + Field::new("p_type", DataType::Utf8, false), + Field::new("p_size", DataType::Int32, false), + Field::new("p_container", DataType::Utf8, false), + Field::new("p_retailprice", DataType::Decimal128(15, 2), false), + Field::new("p_comment", DataType::Utf8, false), + ]), + + "supplier" => datafusion::arrow::datatypes::Schema::new(vec![ + Field::new("s_suppkey", DataType::Int64, false), + Field::new("s_name", DataType::Utf8, false), + Field::new("s_address", DataType::Utf8, false), + Field::new("s_nationkey", DataType::Int64, false), + Field::new("s_phone", DataType::Utf8, false), + Field::new("s_acctbal", DataType::Decimal128(15, 2), false), + Field::new("s_comment", DataType::Utf8, false), + ]), + + "partsupp" => datafusion::arrow::datatypes::Schema::new(vec![ + Field::new("ps_partkey", DataType::Int64, false), + Field::new("ps_suppkey", DataType::Int64, false), + Field::new("ps_availqty", DataType::Int32, false), + Field::new("ps_supplycost", DataType::Decimal128(15, 2), false), + Field::new("ps_comment", DataType::Utf8, false), + ]), + + "customer" => datafusion::arrow::datatypes::Schema::new(vec![ + Field::new("c_custkey", DataType::Int64, false), + Field::new("c_name", DataType::Utf8, false), + Field::new("c_address", DataType::Utf8, false), + Field::new("c_nationkey", DataType::Int64, false), + Field::new("c_phone", DataType::Utf8, false), + Field::new("c_acctbal", DataType::Decimal128(15, 2), false), + Field::new("c_mktsegment", DataType::Utf8, false), + Field::new("c_comment", DataType::Utf8, false), + ]), + + "orders" => datafusion::arrow::datatypes::Schema::new(vec![ + Field::new("o_orderkey", DataType::Int64, false), + Field::new("o_custkey", DataType::Int64, false), + Field::new("o_orderstatus", DataType::Utf8, false), + Field::new("o_totalprice", DataType::Decimal128(15, 2), false), + Field::new("o_orderdate", DataType::Date32, false), + Field::new("o_orderpriority", DataType::Utf8, false), + Field::new("o_clerk", DataType::Utf8, false), + Field::new("o_shippriority", DataType::Int32, false), + Field::new("o_comment", DataType::Utf8, false), + ]), + + "lineitem" => datafusion::arrow::datatypes::Schema::new(vec![ + Field::new("l_orderkey", DataType::Int64, false), + Field::new("l_partkey", DataType::Int64, false), + Field::new("l_suppkey", DataType::Int64, false), + Field::new("l_linenumber", DataType::Int32, false), + Field::new("l_quantity", DataType::Decimal128(15, 2), false), + Field::new("l_extendedprice", DataType::Decimal128(15, 2), false), + Field::new("l_discount", DataType::Decimal128(15, 2), false), + Field::new("l_tax", DataType::Decimal128(15, 2), false), + Field::new("l_returnflag", DataType::Utf8, false), + Field::new("l_linestatus", DataType::Utf8, false), + Field::new("l_shipdate", DataType::Date32, false), + Field::new("l_commitdate", DataType::Date32, false), + Field::new("l_receiptdate", DataType::Date32, false), + Field::new("l_shipinstruct", DataType::Utf8, false), + Field::new("l_shipmode", DataType::Utf8, false), + Field::new("l_comment", DataType::Utf8, false), + ]), + + "nation" => datafusion::arrow::datatypes::Schema::new(vec![ + Field::new("n_nationkey", DataType::Int64, false), + Field::new("n_name", DataType::Utf8, false), + Field::new("n_regionkey", DataType::Int64, false), + Field::new("n_comment", DataType::Utf8, false), + ]), + + "region" => datafusion::arrow::datatypes::Schema::new(vec![ + Field::new("r_regionkey", DataType::Int64, false), + Field::new("r_name", DataType::Utf8, false), + Field::new("r_comment", DataType::Utf8, false), + ]), + + _ => unimplemented!(), + } +} diff --git a/benchmarks/src/run.rs b/benchmarks/src/run.rs index cdbff0a3..5856d32c 100644 --- a/benchmarks/src/run.rs +++ b/benchmarks/src/run.rs @@ -59,7 +59,7 @@ use tonic::transport::Server; pub struct RunOpt { /// Query number. If not specified, runs all queries #[structopt(short, long, use_delimiter = true)] - pub query: Vec, + pub query: Vec, /// Path to data files #[structopt(parse(from_os_str), short = "p", long = "path")] @@ -142,18 +142,20 @@ impl Dataset { } } - fn queries(&self) -> Result, DataFusionError> { + fn queries(&self) -> Result, DataFusionError> { match self { - Dataset::Tpch => (1..22 + 1) - .map(|i| Ok((i as usize, tpch::get_test_tpch_query(i)?))) + Dataset::Tpch => tpch::get_queries() + .into_iter() + .map(|id| Ok((id.clone(), tpch::get_query(&id)?))) .collect(), - Dataset::Tpcds => (1..72) - // skip query 72, it's ridiculously slow - .chain(73..99 + 1) - .map(|i| Ok((i, tpcds::get_test_tpcds_query(i)?))) + Dataset::Tpcds => tpcds::get_queries() + .into_iter() + .filter(|id| id != "q72") // 72 is terribly slow + .map(|id| Ok((id.clone(), tpcds::get_query(&id)?))) .collect(), - Dataset::Clickbench => (0..42 + 1) - .map(|i| Ok((i, clickbench::get_test_clickbench_query(i)?))) + Dataset::Clickbench => clickbench::get_queries() + .into_iter() + .map(|id| Ok((id.clone(), clickbench::get_query(&id)?))) .collect(), } } @@ -226,7 +228,7 @@ impl RunOpt { let dataset = Dataset::infer_from_data_path(path.clone())?; for (id, sql) in dataset.queries()? { - if !self.query.is_empty() && !self.query.contains(&id) { + if !self.query.is_empty() && !self.query.contains(&id.to_string()) { continue; } let query_id = format!("{dataset:?} {id}"); diff --git a/src/test_utils/benchmarks_common.rs b/src/test_utils/benchmarks_common.rs new file mode 100644 index 00000000..5bce41a9 --- /dev/null +++ b/src/test_utils/benchmarks_common.rs @@ -0,0 +1,62 @@ +use datafusion::common::{DataFusionError, internal_datafusion_err, internal_err}; +use datafusion::prelude::{ParquetReadOptions, SessionContext}; +use std::fs; +use std::path::Path; + +pub(crate) fn get_queries(path: &str) -> Vec { + let queries_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(path); + let mut result = vec![]; + for file in queries_dir.read_dir().unwrap() { + let file = file.unwrap(); + let file_name = file.file_name().display().to_string(); + if file_name.ends_with(".sql") { + result.push(file_name.trim_end_matches(".sql").to_string()); + } + } + result +} + +pub(crate) fn get_query(path: &str, id: &str) -> Result { + let queries_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(path); + + if !queries_dir.exists() { + return internal_err!( + "Benchmark queries directory not found: {}", + queries_dir.display() + ); + } + + let query_file = queries_dir.join(format!("{id}.sql")); + + if !query_file.exists() { + return internal_err!("Query file not found: {}", query_file.display()); + } + + let query_sql = fs::read_to_string(&query_file) + .map_err(|e| { + internal_datafusion_err!("Failed to read query file {}: {e}", query_file.display()) + })? + .trim() + .to_string(); + + Ok(query_sql) +} + +pub async fn register_tables( + ctx: &SessionContext, + data_path: &Path, +) -> Result<(), DataFusionError> { + for entry in fs::read_dir(data_path)? { + let path = entry?.path(); + if path.is_dir() { + let table_name = path.file_name().unwrap().to_str().unwrap(); + ctx.register_parquet( + table_name, + path.to_str().unwrap(), + ParquetReadOptions::default(), + ) + .await?; + } + } + Ok(()) +} diff --git a/src/test_utils/clickbench.rs b/src/test_utils/clickbench.rs index fd3259c8..dfa4aad0 100644 --- a/src/test_utils/clickbench.rs +++ b/src/test_utils/clickbench.rs @@ -1,5 +1,5 @@ -use datafusion::common::{DataFusionError, internal_datafusion_err, internal_err}; -use datafusion::prelude::{ParquetReadOptions, SessionContext}; +use crate::test_utils::benchmarks_common; +use datafusion::common::DataFusionError; use std::fs; use std::io::Write; use std::ops::Range; @@ -9,31 +9,12 @@ use tokio::task::JoinSet; const URL: &str = "https://datasets.clickhouse.com/hits_compatible/athena_partitioned/hits_{}.parquet"; -/// Load a single ClickBench query by ID (0-42). -pub fn get_test_clickbench_query(id: usize) -> Result { - let queries_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/clickbench/queries"); - - if !queries_dir.exists() { - return internal_err!( - "TPC-DS queries directory not found: {}", - queries_dir.display() - ); - } - - let query_file = queries_dir.join(format!("q{id}.sql")); - - if !query_file.exists() { - return internal_err!("Query file not found: {}", query_file.display()); - } - - let query_sql = fs::read_to_string(&query_file) - .map_err(|e| { - internal_datafusion_err!("Failed to read query file {}: {e}", query_file.display()) - })? - .trim() - .to_string(); +pub fn get_queries() -> Vec { + benchmarks_common::get_queries("testdata/clickbench/queries") +} - Ok(query_sql) +pub fn get_query(id: &str) -> Result { + benchmarks_common::get_query("testdata/clickbench/queries", id) } /// Downloads the datafusion-benchmarks repository as a zip file @@ -85,22 +66,3 @@ pub async fn generate_clickbench_data( download_partitioned(dest_path.to_path_buf(), range).await?; Ok(()) } - -pub async fn register_tables( - ctx: &SessionContext, - data_path: &Path, -) -> Result<(), DataFusionError> { - for entry in fs::read_dir(data_path)? { - let path = entry?.path(); - if path.is_dir() { - let table_name = path.file_name().unwrap().to_str().unwrap(); - ctx.register_parquet( - table_name, - path.to_str().unwrap(), - ParquetReadOptions::default(), - ) - .await?; - } - } - Ok(()) -} diff --git a/src/test_utils/mod.rs b/src/test_utils/mod.rs index 7b033e12..e565d94f 100644 --- a/src/test_utils/mod.rs +++ b/src/test_utils/mod.rs @@ -1,3 +1,4 @@ +pub mod benchmarks_common; pub mod clickbench; pub mod in_memory_channel_resolver; pub mod insta; diff --git a/src/test_utils/tpcds.rs b/src/test_utils/tpcds.rs index 017eb9bf..5b8a5e16 100644 --- a/src/test_utils/tpcds.rs +++ b/src/test_utils/tpcds.rs @@ -1,5 +1,6 @@ +use crate::test_utils::benchmarks_common; use arrow::datatypes::{DataType, Field}; -use datafusion::common::{internal_datafusion_err, internal_err}; +use datafusion::common::internal_err; use datafusion::error::DataFusionError; use datafusion::physical_expr::Partitioning; use datafusion::physical_expr::expressions::{CastColumnExpr, Column}; @@ -16,31 +17,12 @@ use std::sync::Arc; const URL: &str = "https://github.com/apache/datafusion-benchmarks/archive/refs/heads/main.zip"; -/// Load a single TPC-DS query by ID (1-99). -pub fn get_test_tpcds_query(id: usize) -> Result { - let queries_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/tpcds/queries"); - - if !queries_dir.exists() { - return internal_err!( - "TPC-DS queries directory not found: {}", - queries_dir.display() - ); - } - - let query_file = queries_dir.join(format!("q{id}.sql")); - - if !query_file.exists() { - return internal_err!("Query file not found: {}", query_file.display()); - } - - let query_sql = fs::read_to_string(&query_file) - .map_err(|e| { - internal_datafusion_err!("Failed to read query file {}: {e}", query_file.display()) - })? - .trim() - .to_string(); +pub fn get_queries() -> Vec { + benchmarks_common::get_queries("testdata/tpcds/queries") +} - Ok(query_sql) +pub fn get_query(id: &str) -> Result { + benchmarks_common::get_query("testdata/tpcds/queries", id) } /// Downloads the datafusion-benchmarks repository as a zip file @@ -189,7 +171,7 @@ fn project_cols_as_dict( Ok(Arc::new(project)) } -pub async fn prepare_tables( +async fn prepare_tables( data_path: PathBuf, dest_path: PathBuf, partitions: usize, @@ -217,7 +199,7 @@ pub async fn prepare_tables( Ok(()) } -pub async fn generate_tpcds_data( +pub async fn generate_data( dir: &Path, sf: f64, partitions: usize, @@ -231,22 +213,3 @@ pub async fn generate_tpcds_data( prepare_tables(base_path.join("downloaded"), dir.to_path_buf(), partitions).await?; Ok(()) } - -pub async fn register_tables( - ctx: &SessionContext, - data_path: &Path, -) -> Result<(), DataFusionError> { - for entry in fs::read_dir(data_path)? { - let path = entry?.path(); - if path.is_dir() { - let table_name = path.file_name().unwrap().to_str().unwrap(); - ctx.register_parquet( - table_name, - path.to_str().unwrap(), - ParquetReadOptions::default(), - ) - .await?; - } - } - Ok(()) -} diff --git a/src/test_utils/tpch.rs b/src/test_utils/tpch.rs index ff112dfb..c61b3400 100644 --- a/src/test_utils/tpch.rs +++ b/src/test_utils/tpch.rs @@ -1,16 +1,8 @@ -use std::sync::Arc; - -use datafusion::{ - arrow::datatypes::{DataType, Field, Schema}, - catalog::{MemTable, TableProvider}, -}; - +use crate::test_utils::benchmarks_common; use arrow::record_batch::RecordBatch; -use datafusion::common::not_impl_datafusion_err; use datafusion::error::DataFusionError; use parquet::{arrow::arrow_writer::ArrowWriter, file::properties::WriterProperties}; use std::fs; -use std::path::Path; use tpchgen::generators::{ CustomerGenerator, LineItemGenerator, NationGenerator, OrderGenerator, PartGenerator, PartSuppGenerator, RegionGenerator, SupplierGenerator, @@ -20,118 +12,12 @@ use tpchgen_arrow::{ SupplierArrow, }; -pub fn get_test_tpch_query(num: u8) -> Result { - let queries_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/tpch/queries"); - let query_path = queries_dir.join(format!("q{num}.sql")); - fs::read_to_string(query_path) - .map(|v| v.trim().to_string()) - .map_err(|_| not_impl_datafusion_err!("Failed to read TPCH query file: q{num}.sql")) +pub fn get_queries() -> Vec { + benchmarks_common::get_queries("testdata/tpch/queries") } -pub const NUM_QUERIES: u8 = 22; // number of queries in the TPCH benchmark numbered from 1 to 22 - -pub const TPCH_TABLES: &[&str] = &[ - "part", "supplier", "partsupp", "customer", "orders", "lineitem", "nation", "region", -]; - -pub fn tpch_table(name: &str) -> Arc { - let schema = Arc::new(get_tpch_table_schema(name)); - Arc::new(MemTable::try_new(schema, vec![]).unwrap()) -} - -pub fn get_tpch_table_schema(table: &str) -> Schema { - // note that the schema intentionally uses signed integers so that any generated Parquet - // files can also be used to benchmark tools that only support signed integers, such as - // Apache Spark - - match table { - "part" => Schema::new(vec![ - Field::new("p_partkey", DataType::Int64, false), - Field::new("p_name", DataType::Utf8, false), - Field::new("p_mfgr", DataType::Utf8, false), - Field::new("p_brand", DataType::Utf8, false), - Field::new("p_type", DataType::Utf8, false), - Field::new("p_size", DataType::Int32, false), - Field::new("p_container", DataType::Utf8, false), - Field::new("p_retailprice", DataType::Decimal128(15, 2), false), - Field::new("p_comment", DataType::Utf8, false), - ]), - - "supplier" => Schema::new(vec![ - Field::new("s_suppkey", DataType::Int64, false), - Field::new("s_name", DataType::Utf8, false), - Field::new("s_address", DataType::Utf8, false), - Field::new("s_nationkey", DataType::Int64, false), - Field::new("s_phone", DataType::Utf8, false), - Field::new("s_acctbal", DataType::Decimal128(15, 2), false), - Field::new("s_comment", DataType::Utf8, false), - ]), - - "partsupp" => Schema::new(vec![ - Field::new("ps_partkey", DataType::Int64, false), - Field::new("ps_suppkey", DataType::Int64, false), - Field::new("ps_availqty", DataType::Int32, false), - Field::new("ps_supplycost", DataType::Decimal128(15, 2), false), - Field::new("ps_comment", DataType::Utf8, false), - ]), - - "customer" => Schema::new(vec![ - Field::new("c_custkey", DataType::Int64, false), - Field::new("c_name", DataType::Utf8, false), - Field::new("c_address", DataType::Utf8, false), - Field::new("c_nationkey", DataType::Int64, false), - Field::new("c_phone", DataType::Utf8, false), - Field::new("c_acctbal", DataType::Decimal128(15, 2), false), - Field::new("c_mktsegment", DataType::Utf8, false), - Field::new("c_comment", DataType::Utf8, false), - ]), - - "orders" => Schema::new(vec![ - Field::new("o_orderkey", DataType::Int64, false), - Field::new("o_custkey", DataType::Int64, false), - Field::new("o_orderstatus", DataType::Utf8, false), - Field::new("o_totalprice", DataType::Decimal128(15, 2), false), - Field::new("o_orderdate", DataType::Date32, false), - Field::new("o_orderpriority", DataType::Utf8, false), - Field::new("o_clerk", DataType::Utf8, false), - Field::new("o_shippriority", DataType::Int32, false), - Field::new("o_comment", DataType::Utf8, false), - ]), - - "lineitem" => Schema::new(vec![ - Field::new("l_orderkey", DataType::Int64, false), - Field::new("l_partkey", DataType::Int64, false), - Field::new("l_suppkey", DataType::Int64, false), - Field::new("l_linenumber", DataType::Int32, false), - Field::new("l_quantity", DataType::Decimal128(15, 2), false), - Field::new("l_extendedprice", DataType::Decimal128(15, 2), false), - Field::new("l_discount", DataType::Decimal128(15, 2), false), - Field::new("l_tax", DataType::Decimal128(15, 2), false), - Field::new("l_returnflag", DataType::Utf8, false), - Field::new("l_linestatus", DataType::Utf8, false), - Field::new("l_shipdate", DataType::Date32, false), - Field::new("l_commitdate", DataType::Date32, false), - Field::new("l_receiptdate", DataType::Date32, false), - Field::new("l_shipinstruct", DataType::Utf8, false), - Field::new("l_shipmode", DataType::Utf8, false), - Field::new("l_comment", DataType::Utf8, false), - ]), - - "nation" => Schema::new(vec![ - Field::new("n_nationkey", DataType::Int64, false), - Field::new("n_name", DataType::Utf8, false), - Field::new("n_regionkey", DataType::Int64, false), - Field::new("n_comment", DataType::Utf8, false), - ]), - - "region" => Schema::new(vec![ - Field::new("r_regionkey", DataType::Int64, false), - Field::new("r_name", DataType::Utf8, false), - Field::new("r_comment", DataType::Utf8, false), - ]), - - _ => unimplemented!(), - } +pub fn get_query(id: &str) -> Result { + benchmarks_common::get_query("testdata/tpch/queries", id) } // generate_table creates a parquet file in the data directory from an arrow RecordBatch row diff --git a/tests/clickbench_correctness_test.rs b/tests/clickbench_correctness_test.rs index daae5143..d1165950 100644 --- a/tests/clickbench_correctness_test.rs +++ b/tests/clickbench_correctness_test.rs @@ -5,11 +5,11 @@ mod tests { use datafusion::error::Result; use datafusion::physical_plan::{ExecutionPlan, collect}; use datafusion::prelude::SessionContext; - use datafusion_distributed::test_utils::clickbench; use datafusion_distributed::test_utils::localhost::start_localhost_context; use datafusion_distributed::test_utils::property_based::{ compare_ordering, compare_result_set, }; + use datafusion_distributed::test_utils::{benchmarks_common, clickbench}; use datafusion_distributed::{ DefaultSessionBuilder, DistributedExec, DistributedExt, display_plan_ascii, }; @@ -26,226 +26,226 @@ mod tests { #[tokio::test] #[ignore = "Query 0 did not get distributed"] async fn test_clickbench_0() -> Result<()> { - test_clickbench_query(0).await + test_clickbench_query("q0").await } #[tokio::test] async fn test_clickbench_1() -> Result<()> { - test_clickbench_query(1).await + test_clickbench_query("q1").await } #[tokio::test] async fn test_clickbench_2() -> Result<()> { - test_clickbench_query(2).await + test_clickbench_query("q2").await } #[tokio::test] #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 1, Right set size: 1\n\nRows only in left (1 total):\n 2533767602294735360.00\n\nRows only in right (1 total):\n 2533767602294735872.00.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] async fn test_clickbench_3() -> Result<()> { - test_clickbench_query(3).await + test_clickbench_query("q3").await } #[tokio::test] async fn test_clickbench_4() -> Result<()> { - test_clickbench_query(4).await + test_clickbench_query("q4").await } #[tokio::test] async fn test_clickbench_5() -> Result<()> { - test_clickbench_query(5).await + test_clickbench_query("q5").await } #[tokio::test] #[ignore = "Query 6 did not get distributed"] async fn test_clickbench_6() -> Result<()> { - test_clickbench_query(6).await + test_clickbench_query("q6").await } #[tokio::test] async fn test_clickbench_7() -> Result<()> { - test_clickbench_query(7).await + test_clickbench_query("q7").await } #[tokio::test] async fn test_clickbench_8() -> Result<()> { - test_clickbench_query(8).await + test_clickbench_query("q8").await } #[tokio::test] async fn test_clickbench_9() -> Result<()> { - test_clickbench_query(9).await + test_clickbench_query("q9").await } #[tokio::test] async fn test_clickbench_10() -> Result<()> { - test_clickbench_query(10).await + test_clickbench_query("q10").await } #[tokio::test] async fn test_clickbench_11() -> Result<()> { - test_clickbench_query(11).await + test_clickbench_query("q11").await } #[tokio::test] async fn test_clickbench_12() -> Result<()> { - test_clickbench_query(12).await + test_clickbench_query("q12").await } #[tokio::test] async fn test_clickbench_13() -> Result<()> { - test_clickbench_query(13).await + test_clickbench_query("q13").await } #[tokio::test] async fn test_clickbench_14() -> Result<()> { - test_clickbench_query(14).await + test_clickbench_query("q14").await } #[tokio::test] async fn test_clickbench_15() -> Result<()> { - test_clickbench_query(15).await + test_clickbench_query("q15").await } #[tokio::test] async fn test_clickbench_16() -> Result<()> { - test_clickbench_query(16).await + test_clickbench_query("q16").await } #[tokio::test] #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (10 total):\n 3219866204653196665||4\n 3220056705148678697||11\n 3221898002592879542||1\n 3223026783585713477||23\n 3223839745005575457||116\n 3223839745005575457|d0bcd0bed0b6d0bdd0be20d0bbd0b820d0b220d0bad180d0bed0bad0bed0b4d0b8d180d0bed0b2d0b5d0bbd18cd18820d0b1d180d0bed0b4d18b20d0bdd0b020d181d182d0bed0bbd18b20d0b2d0be20d0b2d0bbd0b0d0b4d0b8d0b2d0bed181d182d0bed0ba20d0b2d0b2d0be|1\n 3223949769615485893||1\n 3226415756450197918||24\n 3226664959488084815||62\n 3227160743723019373||71\n\nRows only in right (10 total):\n 700182585509527889||2\n 724127359630680276|d0b8d0b3d180d18b20d0b820d181d0b5d0b3d0bed0b4d0bdd18f3f|1\n 766120398574852544||1\n 766739966065297239||1\n 783205612738304865||3\n 797289180007803204||2\n 804968013253615745||1\n 830548852254311605||1\n 849024737642146119||1\n 849169469997862534||1.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] async fn test_clickbench_17() -> Result<()> { - test_clickbench_query(17).await + test_clickbench_query("q17").await } #[tokio::test] async fn test_clickbench_18() -> Result<()> { - test_clickbench_query(18).await + test_clickbench_query("q18").await } #[tokio::test] async fn test_clickbench_19() -> Result<()> { - test_clickbench_query(19).await + test_clickbench_query("q19").await } #[tokio::test] async fn test_clickbench_20() -> Result<()> { - test_clickbench_query(20).await + test_clickbench_query("q20").await } #[tokio::test] #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (5 total):\n d181d0bbd0b0d0b2d0bbd18fd182d18c20d0bfd0bed180d0bed0b4d0b8d182d181d18f20d0bed182d0b5d0bbd0b8203230313320d181d0bcd0bed182d180d0b5d182d18c|687474703a253246253246766b2e636f6d2e75612f676f6f676c652d6a61726b6f76736b6179612d4c697065636b64|1\n d0b1d0b0d0bdd0bad0bed0bcd0b0d182d0b5d180d0b8d0b0d0bbd18b20d181d0bcd0bed182d180d0b5d182d18c|687474703a2f2f6f72656e627572672e6972722e72752532466b7572746b692532462532467777772e676f6f676c652e72752f6d617a64612d332d6b6f6d6e2d6b762d4b617a616e2e74757475746f72736b2f64657461696c|1\n d0bcd0bed0bdd0b8d182d18c20d0bad0b0d0bad0bed0b520d0bed0b7d0b5d180d0b0|687474703a2f2f6175746f2e7269612e75612f6175746f5f69643d30266f726465723d46616c7365266d696e707269782e72752f6b617465676f726979612f767369652d646c69612d647275676f652f6d61746572696e7374766f2f676f6f676c652d706f6c697331343334343532|1\n d181d0bad0b0d187d0b0d182d18c20d0b4d0b5d0bdd0b5d0b320d181d183d180d0b3d183d182|687474703a2f2f7469656e736b6169612d6d6f64612d627269657469656c6b612d6b6f736b6f76736b2f64657461696c2e676f6f676c65|1\n d0b220d0b0d0b2d0b3d183d181d1822032343720d0b3d180d183d181d182d0b8d0bcd0bed188d0bad0b020d0bdd0b020d0bad180d0b8d181d182d180d0b0d182|687474703a2f2f7469656e736b6169612d6d6f64612d627269756b692f676f6f676c652e72752f7e61706f6b2e72752f635f312d755f313138383839352c39373536|1\n\nRows only in right (5 total):\n d0bcd0bed0b4d0b5d0bad18120d183d0bbd0b8d186d0b5d0bdd0b7d0b8d0bdd0bed0b2d0b020d0b3d0bed0b2d18fd0b4d0b8d0bdd0b0|687474703a2f2f73616d6172612e6972722e72752f636174616c6f675f676f6f676c652d636865726e796a2d393233353636363635372f3f64617465|1\n d0bbd0b0d0b2d0bfd0bbd0b0d0bdd188d0b5d182d0bdd0b8d18520d183d181d0bbd0bed0b2d0b0d0bcd0b820d0b2d181d0b520d181d0b5d180d0b8d0b820d0b4d0b0d182d0b020d186d0b5d0bcd0b5d0bdd0b8|687474703a2f2f73616d6172612e6972722e72752f636174616c6f675f676f6f676c654d425225323661642533443930253236707a|1\n d0bad0b0d0ba20d0bfd180d0bed0b4d0b0d0bcd0b820d0b4d0bbd18f20d0b4d0b5d0b2d183d188d0bad0b8|687474703a253246253246777777772e626f6e707269782e7275253235326625323532663737363925323532663131303931392d6c65766f652d676f6f676c652d7368746f72792e72752f666f72756d2f666f72756d2e6d617465722e72752f6461696c792f63616c63756c61746f72|1\n d0b6d0b0d180d0b5d0bdd18cd18f20d0b32ed181d183d180d0bed0b2d0b0d0bdd0b8d0b520d0b2d0bed180d0bed0bdd0b5d0b6d181d0bad0b0d18f20d0bed0b1d0bbd0b0d181d182d0bed0bfd180d0b8d0bbd0b520d0bfd0bed181d0bbd0b5d0b4d0bdd0b8d0b520d0bad0bed181d18b|687474703a2f2f756b7261696e627572672f65636f2d6d6c656b2f65636f6e646172792f73686f77746f7069632e7068703f69643d3436333837362e68746d6c3f69643d32303634313333363631253246676f6f676c652d4170706c655765624b69742532463533372e333620284b48544d4c2c206c696b65|1\n d180d0b8d0be20d0bdd0b020d0bad0b0d180d182d0bed187d0bdd0b8d186d0b020d181d0bcd0bed182d180d0b5d182d18c20d0bed0bdd0bbd0b0d0b9d0bd|687474703a2f2f73616d6172612e6972722e72752f636174616c6f675f676f6f676c654d425225323661642533443930253236707a|1.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] async fn test_clickbench_21() -> Result<()> { - test_clickbench_query(21).await + test_clickbench_query("q21").await } #[tokio::test] #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (1 total):\n d0bad0b0d0bad0bed0b920d0bfd0bbd0bed189d0b0d0b4d0bad0b8d0bcd0b820d0b4d0bed181d182d0b0d0b2d0bad0b8|687474703a253246253246766b2e636f6d2f696672616d652d6f77612e68746d6c3f313d31266369643d353737266f6b693d31266f705f63617465676f72795f69645d3d332673656c656374|d092d0b0d0bad0b0d0bdd181d0b8d18f20d091d0a0d090d09ad090d09d20d090d09dd094d0a0d095d0a1202d20d0bfd0bed0bfd0b0d0bbd0b820d0bad183d0bfd0b8d182d18c20d0b4d0bed0bcd0bed0b5d187d0bdd18bd0b520d188d0bad0b0d184d0b020476f6f676c652e636f6d203a3a20d0bad0bed182d182d0b5d0bad181d1822c20d091d183d180d18fd182d0bdd0b8d0bad0b820d0b4d0bbd18f20d0bfd0b5d187d18c20d0bcd0b5d0b1d0b5d0bbd18cd0b520d0b4d0bbd18f20d0b4d0b5d0b2d183d188d0bad0b0|5|1\n\nRows only in right (1 total):\n d0bad0bed0bfd182d0b8d0bcd0b8d0bad0b2d0b8d0b4d18b20d18ed180d0b8d0b920d0bfd0bed181d0bbd0b5d0b4d0bdd18fd18f|68747470733a2f2f70726f64756b747925324670756c6f76652e72752f626f6f6b6c79617474696f6e2d7761722d73696e696a2d393430343139342c3936323435332f666f746f|d09bd0b5d0b3d0bad0be20d0bdd0b020d183d187d0b0d181d182d0bdd18bd0b520d183d187d0b0d181d182d0bdd0b8d0bad0bed0b22e2c20d0a6d0b5d0bdd18b202d20d0a1d182d0b8d0bbd18cd0bdd0b0d18f20d0bfd0b0d180d0bdd0b5d0bc2e20d0a1d0b0d0b3d0b0d0bdd180d0bed0b320d0b4d0bed0b3d0b0d0b4d0b5d0bdd0b8d18f203a20d0a2d183d180d186d0b8d0b82c20d0bad183d0bfd0b8d182d18c20d18320313020d0b4d0bdd0b520d0bad0bed0bbd18cd0bdd18bd0b520d0bcd0b0d188d0b8d0bdd0bad0b820d0bdd0b520d0bfd180d0b5d0b4d181d182d0b0d0b2d0bad0b8202d20d09dd0bed0b2d0b0d18f20d18120d0b8d0b7d0b1d0b8d0b5d0bdd0b8d0b520d181d0bfd180d0bed0b4d0b0d0b6d0b03a20d0bad0bed182d18fd182d0b0203230313420d0b32ed0b22e20d0a6d0b5d0bdd0b03a2034373530302d313045434f30363020e28093202d2d2d2d2d2d2d2d20d0bad183d0bfd0b8d182d18c20d0bad0b2d0b0d180d182d0b8d180d18320d09ed180d0b5d0bdd0b1d183d180d0b32028d0a0d0bed181d181d0b8d0b82047616c616e7472617820466c616d696c6961646120476f6f676c652c204ed0be20313820d184d0bed182d0bed0bad0bed0bdd0b2d0b5d180d0ba20d0a1d183d0bfd0b5d18020d09ad0b0d180d0b4d0b8d0b3d0b0d0bd|5|1.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] async fn test_clickbench_22() -> Result<()> { - test_clickbench_query(22).await + test_clickbench_query("q22").await } #[tokio::test] async fn test_clickbench_23() -> Result<()> { - test_clickbench_query(23).await + test_clickbench_query("q23").await } #[tokio::test] #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (1 total):\n d0b2d181d0bfd0bed0bcd0bdd0b8d182d18c20d181d0bed0bbd0bdd0b5d0bdd0b8d0b520d0b1d0b0d0bdd0bad0b020d0bbd0b0d0b420d184d0b8d0bbd18cd0bc\n\nRows only in right (1 total):\n d0bed182d0b2d0bed0b4d0b020d0b4d0bbd18f20d0bfd0b8d180d0bed0b6d0bad0b820d0bbd0b5d187d0b5d0bdd0bdd18b20d0b2d181d0b520d181d0b5d180d196d197.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] async fn test_clickbench_24() -> Result<()> { - test_clickbench_query(24).await + test_clickbench_query("q24").await } #[tokio::test] async fn test_clickbench_25() -> Result<()> { - test_clickbench_query(25).await + test_clickbench_query("q25").await } #[tokio::test] async fn test_clickbench_26() -> Result<()> { - test_clickbench_query(26).await + test_clickbench_query("q26").await } #[tokio::test] async fn test_clickbench_27() -> Result<()> { - test_clickbench_query(27).await + test_clickbench_query("q27").await } #[tokio::test] async fn test_clickbench_28() -> Result<()> { - test_clickbench_query(28).await + test_clickbench_query("q28").await } #[tokio::test] async fn test_clickbench_29() -> Result<()> { - test_clickbench_query(29).await + test_clickbench_query("q29").await } #[tokio::test] async fn test_clickbench_30() -> Result<()> { - test_clickbench_query(30).await + test_clickbench_query("q30").await } #[tokio::test] #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (10 total):\n 8673025726158767406|1264438551|1|0|1990.00\n 5320052218057629211|-1703087277|1|0|1996.00\n 6244273852606083750|1554672832|1|0|1638.00\n 8628753750962053665|1215278356|1|0|1087.00\n 7035318163404387241|1326714320|1|0|1638.00\n 8431857775494210873|1237512945|1|0|1996.00\n 5110752526539992124|37611695|1|0|1917.00\n 8986794334343068049|1860752926|1|0|1638.00\n 8044147848299485837|1382122372|1|0|1368.00\n 7936057634954670727|1897481896|1|0|1638.00\n\nRows only in right (10 total):\n 5132615111782210132|-50313020|1|0|1368.00\n 5783789691451717551|-1310327384|1|0|1638.00\n 5756260993772351383|1484317883|1|0|375.00\n 7739310142000732364|991864113|1|0|1368.00\n 7593472904893539271|-151291403|1|0|1087.00\n 6339599967989898410|1543815587|1|0|1638.00\n 7794346560421945218|1645556180|1|0|1368.00\n 6112645108657361792|593586188|1|0|1638.00\n 6675910710751922756|-816379256|1|0|1368.00\n 5802727636196431835|1986422271|1|0|1996.00.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] async fn test_clickbench_31() -> Result<()> { - test_clickbench_query(31).await + test_clickbench_query("q31").await } #[tokio::test] #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (10 total):\n 7643059318918524417|1767085700|1|0|0.00\n 5437163248266133938|-1465369615|1|0|0.00\n 9142541582422390102|-1465369615|1|0|0.00\n 8438994503411842126|-1465369615|1|0|0.00\n 7362096505818029859|-565678477|1|0|0.00\n 4928022308880516715|1699955284|1|0|0.00\n 5269769817689282522|1699955284|1|0|0.00\n 9081648050908046886|1699955284|1|0|0.00\n 6824181869275536503|1699955284|1|0|0.00\n 6905712404475757487|1552811156|1|0|0.00\n\nRows only in right (10 total):\n 6967277596165459879|-941091661|1|0|1368.00\n 5796237228224217668|-1310327384|1|0|1638.00\n 7218628137278606666|1511490240|1|0|1638.00\n 8314760197723815280|1566105210|1|0|1996.00\n 7053263954762394007|757778490|1|0|339.00\n 6283334114093174531|1216031795|1|0|1368.00\n 8818295356247036741|83042182|1|0|1638.00\n 6620528864937282562|-862894777|1|0|1996.00\n 8466121050002905379|83042182|1|0|1638.00\n 7554844936512227411|-1746904856|1|0|1368.00.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] async fn test_clickbench_32() -> Result<()> { - test_clickbench_query(32).await + test_clickbench_query("q32").await } #[tokio::test] async fn test_clickbench_33() -> Result<()> { - test_clickbench_query(33).await + test_clickbench_query("q33").await } #[tokio::test] async fn test_clickbench_34() -> Result<()> { - test_clickbench_query(34).await + test_clickbench_query("q34").await } #[tokio::test] async fn test_clickbench_35() -> Result<()> { - test_clickbench_query(35).await + test_clickbench_query("q35").await } #[tokio::test] async fn test_clickbench_36() -> Result<()> { - test_clickbench_query(36).await + test_clickbench_query("q36").await } #[tokio::test] async fn test_clickbench_37() -> Result<()> { - test_clickbench_query(37).await + test_clickbench_query("q37").await } #[tokio::test] async fn test_clickbench_38() -> Result<()> { - test_clickbench_query(38).await + test_clickbench_query("q38").await } #[tokio::test] async fn test_clickbench_39() -> Result<()> { - test_clickbench_query(39).await + test_clickbench_query("q39").await } #[tokio::test] async fn test_clickbench_40() -> Result<()> { - test_clickbench_query(40).await + test_clickbench_query("q40").await } #[tokio::test] async fn test_clickbench_41() -> Result<()> { - test_clickbench_query(41).await + test_clickbench_query("q41").await } #[tokio::test] #[ignore = "ordering mismatch: expected ordering: Some(LexOrdering { exprs: [PhysicalSortExpr { expr: ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }, options: SortOptions { descending: false, nulls_first: false } }], set: {ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }} }), actual ordering: Some(LexOrdering { exprs: [PhysicalSortExpr { expr: ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }, options: SortOptions { descending: false, nulls_first: false } }], set: {ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }} })"] async fn test_clickbench_42() -> Result<()> { - test_clickbench_query(42).await + test_clickbench_query("q42").await } static INIT_TEST_TPCDS_TABLES: OnceCell<()> = OnceCell::const_new(); @@ -260,7 +260,7 @@ mod tests { (plan.clone(), Arc::new(collect(plan, task_ctx).await)) // Collect execution errors, do not unwrap. } - async fn test_clickbench_query(query_id: usize) -> Result<()> { + async fn test_clickbench_query(query_id: &str) -> Result<()> { let data_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(format!( "testdata/clickbench/correctness_range{}-{}", FILE_RANGE.start, FILE_RANGE.end @@ -273,7 +273,7 @@ mod tests { }) .await; - let query_sql = clickbench::get_test_clickbench_query(query_id)?; + let query_sql = clickbench::get_query(query_id)?; // Create a single node context to compare results to. let s_ctx = SessionContext::new(); @@ -283,8 +283,8 @@ mod tests { .with_distributed_files_per_task(FILES_PER_TASK)? .with_distributed_cardinality_effect_task_scale_factor(CARDINALITY_TASK_COUNT_FACTOR)?; - clickbench::register_tables(&s_ctx, &data_dir).await?; - clickbench::register_tables(&d_ctx, &data_dir).await?; + benchmarks_common::register_tables(&s_ctx, &data_dir).await?; + benchmarks_common::register_tables(&d_ctx, &data_dir).await?; let (s_plan, s_results) = run(&s_ctx, &query_sql).await; let (d_plan, d_results) = run(&d_ctx, &query_sql).await; diff --git a/tests/clickbench_plans_test.rs b/tests/clickbench_plans_test.rs index e9e07432..36528584 100644 --- a/tests/clickbench_plans_test.rs +++ b/tests/clickbench_plans_test.rs @@ -1,8 +1,8 @@ #[cfg(all(feature = "integration", feature = "clickbench", test))] mod tests { use datafusion::error::Result; - use datafusion_distributed::test_utils::clickbench; use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::test_utils::{benchmarks_common, clickbench}; use datafusion_distributed::{ DefaultSessionBuilder, DistributedExec, DistributedExt, assert_snapshot, display_plan_ascii, }; @@ -18,14 +18,14 @@ mod tests { #[tokio::test] #[ignore = "Query 0 did not get distributed"] async fn test_clickbench_0() -> Result<()> { - let display = test_clickbench_query(0).await?; + let display = test_clickbench_query("q0").await?; assert_snapshot!(display, @""); Ok(()) } #[tokio::test] async fn test_clickbench_1() -> Result<()> { - let display = test_clickbench_query(1).await?; + let display = test_clickbench_query("q1").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] @@ -47,7 +47,7 @@ mod tests { #[tokio::test] async fn test_clickbench_2() -> Result<()> { - let display = test_clickbench_query(2).await?; + let display = test_clickbench_query("q2").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[sum(hits.AdvEngineID)@0 as sum(hits.AdvEngineID), count(Int64(1))@1 as count(*), avg(hits.ResolutionWidth)@2 as avg(hits.ResolutionWidth)] @@ -67,7 +67,7 @@ mod tests { #[tokio::test] #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 1, Right set size: 1\n\nRows only in left (1 total):\n 2533767602294735360.00\n\nRows only in right (1 total):\n 2533767602294735872.00.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] async fn test_clickbench_3() -> Result<()> { - let display = test_clickbench_query(3).await?; + let display = test_clickbench_query("q3").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ AggregateExec: mode=Final, gby=[], aggr=[avg(hits.UserID)] @@ -85,7 +85,7 @@ mod tests { #[tokio::test] async fn test_clickbench_4() -> Result<()> { - let display = test_clickbench_query(4).await?; + let display = test_clickbench_query("q4").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[count(alias1)@0 as count(DISTINCT hits.UserID)] @@ -111,7 +111,7 @@ mod tests { #[tokio::test] async fn test_clickbench_5() -> Result<()> { - let display = test_clickbench_query(5).await?; + let display = test_clickbench_query("q5").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[count(alias1)@0 as count(DISTINCT hits.SearchPhrase)] @@ -138,14 +138,14 @@ mod tests { #[tokio::test] #[ignore = "Query 6 did not get distributed"] async fn test_clickbench_6() -> Result<()> { - let display = test_clickbench_query(6).await?; + let display = test_clickbench_query("q6").await?; assert_snapshot!(display, @""); Ok(()) } #[tokio::test] async fn test_clickbench_7() -> Result<()> { - let display = test_clickbench_query(7).await?; + let display = test_clickbench_query("q7").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[AdvEngineID@0 as AdvEngineID, count(*)@1 as count(*)] @@ -173,7 +173,7 @@ mod tests { #[tokio::test] async fn test_clickbench_8() -> Result<()> { - let display = test_clickbench_query(8).await?; + let display = test_clickbench_query("q8").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [u@1 DESC], fetch=10 @@ -202,7 +202,7 @@ mod tests { #[tokio::test] async fn test_clickbench_9() -> Result<()> { - let display = test_clickbench_query(9).await?; + let display = test_clickbench_query("q9").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [c@2 DESC], fetch=10 @@ -227,7 +227,7 @@ mod tests { #[tokio::test] async fn test_clickbench_10() -> Result<()> { - let display = test_clickbench_query(10).await?; + let display = test_clickbench_query("q10").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [u@1 DESC], fetch=10 @@ -258,7 +258,7 @@ mod tests { #[tokio::test] async fn test_clickbench_11() -> Result<()> { - let display = test_clickbench_query(11).await?; + let display = test_clickbench_query("q11").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [u@2 DESC], fetch=10 @@ -289,7 +289,7 @@ mod tests { #[tokio::test] async fn test_clickbench_12() -> Result<()> { - let display = test_clickbench_query(12).await?; + let display = test_clickbench_query("q12").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [c@1 DESC], fetch=10 @@ -316,7 +316,7 @@ mod tests { #[tokio::test] async fn test_clickbench_13() -> Result<()> { - let display = test_clickbench_query(13).await?; + let display = test_clickbench_query("q13").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [u@1 DESC], fetch=10 @@ -347,7 +347,7 @@ mod tests { #[tokio::test] async fn test_clickbench_14() -> Result<()> { - let display = test_clickbench_query(14).await?; + let display = test_clickbench_query("q14").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [c@2 DESC], fetch=10 @@ -374,7 +374,7 @@ mod tests { #[tokio::test] async fn test_clickbench_15() -> Result<()> { - let display = test_clickbench_query(15).await?; + let display = test_clickbench_query("q15").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[UserID@0 as UserID, count(*)@1 as count(*)] @@ -400,7 +400,7 @@ mod tests { #[tokio::test] async fn test_clickbench_16() -> Result<()> { - let display = test_clickbench_query(16).await?; + let display = test_clickbench_query("q16").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[UserID@0 as UserID, SearchPhrase@1 as SearchPhrase, count(*)@2 as count(*)] @@ -427,14 +427,14 @@ mod tests { #[tokio::test] #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (10 total):\n 3219866204653196665||4\n 3220056705148678697||11\n 3221898002592879542||1\n 3223026783585713477||23\n 3223839745005575457||116\n 3223839745005575457|d0bcd0bed0b6d0bdd0be20d0bbd0b820d0b220d0bad180d0bed0bad0bed0b4d0b8d180d0bed0b2d0b5d0bbd18cd18820d0b1d180d0bed0b4d18b20d0bdd0b020d181d182d0bed0bbd18b20d0b2d0be20d0b2d0bbd0b0d0b4d0b8d0b2d0bed181d182d0bed0ba20d0b2d0b2d0be|1\n 3223949769615485893||1\n 3226415756450197918||24\n 3226664959488084815||62\n 3227160743723019373||71\n\nRows only in right (10 total):\n 700182585509527889||2\n 724127359630680276|d0b8d0b3d180d18b20d0b820d181d0b5d0b3d0bed0b4d0bdd18f3f|1\n 766120398574852544||1\n 766739966065297239||1\n 783205612738304865||3\n 797289180007803204||2\n 804968013253615745||1\n 830548852254311605||1\n 849024737642146119||1\n 849169469997862534||1.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] async fn test_clickbench_17() -> Result<()> { - let display = test_clickbench_query(17).await?; + let display = test_clickbench_query("q17").await?; assert_snapshot!(display, @""); Ok(()) } #[tokio::test] async fn test_clickbench_18() -> Result<()> { - let display = test_clickbench_query(18).await?; + let display = test_clickbench_query("q18").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[UserID@0 as UserID, m@1 as m, SearchPhrase@2 as SearchPhrase, count(*)@3 as count(*)] @@ -460,7 +460,7 @@ mod tests { #[tokio::test] async fn test_clickbench_19() -> Result<()> { - let display = test_clickbench_query(19).await?; + let display = test_clickbench_query("q19").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ CoalescePartitionsExec @@ -478,7 +478,7 @@ mod tests { #[tokio::test] async fn test_clickbench_20() -> Result<()> { - let display = test_clickbench_query(20).await?; + let display = test_clickbench_query("q20").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] @@ -501,7 +501,7 @@ mod tests { #[tokio::test] #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (5 total):\n d181d0bbd0b0d0b2d0bbd18fd182d18c20d0bfd0bed180d0bed0b4d0b8d182d181d18f20d0bed182d0b5d0bbd0b8203230313320d181d0bcd0bed182d180d0b5d182d18c|687474703a253246253246766b2e636f6d2e75612f676f6f676c652d6a61726b6f76736b6179612d4c697065636b64|1\n d0b1d0b0d0bdd0bad0bed0bcd0b0d182d0b5d180d0b8d0b0d0bbd18b20d181d0bcd0bed182d180d0b5d182d18c|687474703a2f2f6f72656e627572672e6972722e72752532466b7572746b692532462532467777772e676f6f676c652e72752f6d617a64612d332d6b6f6d6e2d6b762d4b617a616e2e74757475746f72736b2f64657461696c|1\n d0bcd0bed0bdd0b8d182d18c20d0bad0b0d0bad0bed0b520d0bed0b7d0b5d180d0b0|687474703a2f2f6175746f2e7269612e75612f6175746f5f69643d30266f726465723d46616c7365266d696e707269782e72752f6b617465676f726979612f767369652d646c69612d647275676f652f6d61746572696e7374766f2f676f6f676c652d706f6c697331343334343532|1\n d181d0bad0b0d187d0b0d182d18c20d0b4d0b5d0bdd0b5d0b320d181d183d180d0b3d183d182|687474703a2f2f7469656e736b6169612d6d6f64612d627269657469656c6b612d6b6f736b6f76736b2f64657461696c2e676f6f676c65|1\n d0b220d0b0d0b2d0b3d183d181d1822032343720d0b3d180d183d181d182d0b8d0bcd0bed188d0bad0b020d0bdd0b020d0bad180d0b8d181d182d180d0b0d182|687474703a2f2f7469656e736b6169612d6d6f64612d627269756b692f676f6f676c652e72752f7e61706f6b2e72752f635f312d755f313138383839352c39373536|1\n\nRows only in right (5 total):\n d0bcd0bed0b4d0b5d0bad18120d183d0bbd0b8d186d0b5d0bdd0b7d0b8d0bdd0bed0b2d0b020d0b3d0bed0b2d18fd0b4d0b8d0bdd0b0|687474703a2f2f73616d6172612e6972722e72752f636174616c6f675f676f6f676c652d636865726e796a2d393233353636363635372f3f64617465|1\n d0bbd0b0d0b2d0bfd0bbd0b0d0bdd188d0b5d182d0bdd0b8d18520d183d181d0bbd0bed0b2d0b0d0bcd0b820d0b2d181d0b520d181d0b5d180d0b8d0b820d0b4d0b0d182d0b020d186d0b5d0bcd0b5d0bdd0b8|687474703a2f2f73616d6172612e6972722e72752f636174616c6f675f676f6f676c654d425225323661642533443930253236707a|1\n d0bad0b0d0ba20d0bfd180d0bed0b4d0b0d0bcd0b820d0b4d0bbd18f20d0b4d0b5d0b2d183d188d0bad0b8|687474703a253246253246777777772e626f6e707269782e7275253235326625323532663737363925323532663131303931392d6c65766f652d676f6f676c652d7368746f72792e72752f666f72756d2f666f72756d2e6d617465722e72752f6461696c792f63616c63756c61746f72|1\n d0b6d0b0d180d0b5d0bdd18cd18f20d0b32ed181d183d180d0bed0b2d0b0d0bdd0b8d0b520d0b2d0bed180d0bed0bdd0b5d0b6d181d0bad0b0d18f20d0bed0b1d0bbd0b0d181d182d0bed0bfd180d0b8d0bbd0b520d0bfd0bed181d0bbd0b5d0b4d0bdd0b8d0b520d0bad0bed181d18b|687474703a2f2f756b7261696e627572672f65636f2d6d6c656b2f65636f6e646172792f73686f77746f7069632e7068703f69643d3436333837362e68746d6c3f69643d32303634313333363631253246676f6f676c652d4170706c655765624b69742532463533372e333620284b48544d4c2c206c696b65|1\n d180d0b8d0be20d0bdd0b020d0bad0b0d180d182d0bed187d0bdd0b8d186d0b020d181d0bcd0bed182d180d0b5d182d18c20d0bed0bdd0bbd0b0d0b9d0bd|687474703a2f2f73616d6172612e6972722e72752f636174616c6f675f676f6f676c654d425225323661642533443930253236707a|1.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] async fn test_clickbench_21() -> Result<()> { - let display = test_clickbench_query(21).await?; + let display = test_clickbench_query("q21").await?; assert_snapshot!(display, @""); Ok(()) } @@ -509,14 +509,14 @@ mod tests { #[tokio::test] #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (1 total):\n d0bad0b0d0bad0bed0b920d0bfd0bbd0bed189d0b0d0b4d0bad0b8d0bcd0b820d0b4d0bed181d182d0b0d0b2d0bad0b8|687474703a253246253246766b2e636f6d2f696672616d652d6f77612e68746d6c3f313d31266369643d353737266f6b693d31266f705f63617465676f72795f69645d3d332673656c656374|d092d0b0d0bad0b0d0bdd181d0b8d18f20d091d0a0d090d09ad090d09d20d090d09dd094d0a0d095d0a1202d20d0bfd0bed0bfd0b0d0bbd0b820d0bad183d0bfd0b8d182d18c20d0b4d0bed0bcd0bed0b5d187d0bdd18bd0b520d188d0bad0b0d184d0b020476f6f676c652e636f6d203a3a20d0bad0bed182d182d0b5d0bad181d1822c20d091d183d180d18fd182d0bdd0b8d0bad0b820d0b4d0bbd18f20d0bfd0b5d187d18c20d0bcd0b5d0b1d0b5d0bbd18cd0b520d0b4d0bbd18f20d0b4d0b5d0b2d183d188d0bad0b0|5|1\n\nRows only in right (1 total):\n d0bad0bed0bfd182d0b8d0bcd0b8d0bad0b2d0b8d0b4d18b20d18ed180d0b8d0b920d0bfd0bed181d0bbd0b5d0b4d0bdd18fd18f|68747470733a2f2f70726f64756b747925324670756c6f76652e72752f626f6f6b6c79617474696f6e2d7761722d73696e696a2d393430343139342c3936323435332f666f746f|d09bd0b5d0b3d0bad0be20d0bdd0b020d183d187d0b0d181d182d0bdd18bd0b520d183d187d0b0d181d182d0bdd0b8d0bad0bed0b22e2c20d0a6d0b5d0bdd18b202d20d0a1d182d0b8d0bbd18cd0bdd0b0d18f20d0bfd0b0d180d0bdd0b5d0bc2e20d0a1d0b0d0b3d0b0d0bdd180d0bed0b320d0b4d0bed0b3d0b0d0b4d0b5d0bdd0b8d18f203a20d0a2d183d180d186d0b8d0b82c20d0bad183d0bfd0b8d182d18c20d18320313020d0b4d0bdd0b520d0bad0bed0bbd18cd0bdd18bd0b520d0bcd0b0d188d0b8d0bdd0bad0b820d0bdd0b520d0bfd180d0b5d0b4d181d182d0b0d0b2d0bad0b8202d20d09dd0bed0b2d0b0d18f20d18120d0b8d0b7d0b1d0b8d0b5d0bdd0b8d0b520d181d0bfd180d0bed0b4d0b0d0b6d0b03a20d0bad0bed182d18fd182d0b0203230313420d0b32ed0b22e20d0a6d0b5d0bdd0b03a2034373530302d313045434f30363020e28093202d2d2d2d2d2d2d2d20d0bad183d0bfd0b8d182d18c20d0bad0b2d0b0d180d182d0b8d180d18320d09ed180d0b5d0bdd0b1d183d180d0b32028d0a0d0bed181d181d0b8d0b82047616c616e7472617820466c616d696c6961646120476f6f676c652c204ed0be20313820d184d0bed182d0bed0bad0bed0bdd0b2d0b5d180d0ba20d0a1d183d0bfd0b5d18020d09ad0b0d180d0b4d0b8d0b3d0b0d0bd|5|1.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] async fn test_clickbench_22() -> Result<()> { - let display = test_clickbench_query(22).await?; + let display = test_clickbench_query("q22").await?; assert_snapshot!(display, @""); Ok(()) } #[tokio::test] async fn test_clickbench_23() -> Result<()> { - let display = test_clickbench_query(23).await?; + let display = test_clickbench_query("q23").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [EventTime@4 ASC NULLS LAST], fetch=10 @@ -536,14 +536,14 @@ mod tests { #[tokio::test] #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (1 total):\n d0b2d181d0bfd0bed0bcd0bdd0b8d182d18c20d181d0bed0bbd0bdd0b5d0bdd0b8d0b520d0b1d0b0d0bdd0bad0b020d0bbd0b0d0b420d184d0b8d0bbd18cd0bc\n\nRows only in right (1 total):\n d0bed182d0b2d0bed0b4d0b020d0b4d0bbd18f20d0bfd0b8d180d0bed0b6d0bad0b820d0bbd0b5d187d0b5d0bdd0bdd18b20d0b2d181d0b520d181d0b5d180d196d197.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] async fn test_clickbench_24() -> Result<()> { - let display = test_clickbench_query(24).await?; + let display = test_clickbench_query("q24").await?; assert_snapshot!(display, @""); Ok(()) } #[tokio::test] async fn test_clickbench_25() -> Result<()> { - let display = test_clickbench_query(25).await?; + let display = test_clickbench_query("q25").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [SearchPhrase@0 ASC NULLS LAST], fetch=10 @@ -562,7 +562,7 @@ mod tests { #[tokio::test] async fn test_clickbench_26() -> Result<()> { - let display = test_clickbench_query(26).await?; + let display = test_clickbench_query("q26").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase] @@ -583,7 +583,7 @@ mod tests { #[tokio::test] async fn test_clickbench_27() -> Result<()> { - let display = test_clickbench_query(27).await?; + let display = test_clickbench_query("q27").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [l@1 DESC], fetch=25 @@ -612,7 +612,7 @@ mod tests { #[tokio::test] async fn test_clickbench_28() -> Result<()> { - let display = test_clickbench_query(28).await?; + let display = test_clickbench_query("q28").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [l@1 DESC], fetch=25 @@ -641,7 +641,7 @@ mod tests { #[tokio::test] async fn test_clickbench_29() -> Result<()> { - let display = test_clickbench_query(29).await?; + let display = test_clickbench_query("q29").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ AggregateExec: mode=Final, gby=[], aggr=[sum(hits.ResolutionWidth), sum(hits.ResolutionWidth + Int64(1)), sum(hits.ResolutionWidth + Int64(2)), sum(hits.ResolutionWidth + Int64(3)), sum(hits.ResolutionWidth + Int64(4)), sum(hits.ResolutionWidth + Int64(5)), sum(hits.ResolutionWidth + Int64(6)), sum(hits.ResolutionWidth + Int64(7)), sum(hits.ResolutionWidth + Int64(8)), sum(hits.ResolutionWidth + Int64(9)), sum(hits.ResolutionWidth + Int64(10)), sum(hits.ResolutionWidth + Int64(11)), sum(hits.ResolutionWidth + Int64(12)), sum(hits.ResolutionWidth + Int64(13)), sum(hits.ResolutionWidth + Int64(14)), sum(hits.ResolutionWidth + Int64(15)), sum(hits.ResolutionWidth + Int64(16)), sum(hits.ResolutionWidth + Int64(17)), sum(hits.ResolutionWidth + Int64(18)), sum(hits.ResolutionWidth + Int64(19)), sum(hits.ResolutionWidth + Int64(20)), sum(hits.ResolutionWidth + Int64(21)), sum(hits.ResolutionWidth + Int64(22)), sum(hits.ResolutionWidth + Int64(23)), sum(hits.ResolutionWidth + Int64(24)), sum(hits.ResolutionWidth + Int64(25)), sum(hits.ResolutionWidth + Int64(26)), sum(hits.ResolutionWidth + Int64(27)), sum(hits.ResolutionWidth + Int64(28)), sum(hits.ResolutionWidth + Int64(29)), sum(hits.ResolutionWidth + Int64(30)), sum(hits.ResolutionWidth + Int64(31)), sum(hits.ResolutionWidth + Int64(32)), sum(hits.ResolutionWidth + Int64(33)), sum(hits.ResolutionWidth + Int64(34)), sum(hits.ResolutionWidth + Int64(35)), sum(hits.ResolutionWidth + Int64(36)), sum(hits.ResolutionWidth + Int64(37)), sum(hits.ResolutionWidth + Int64(38)), sum(hits.ResolutionWidth + Int64(39)), sum(hits.ResolutionWidth + Int64(40)), sum(hits.ResolutionWidth + Int64(41)), sum(hits.ResolutionWidth + Int64(42)), sum(hits.ResolutionWidth + Int64(43)), sum(hits.ResolutionWidth + Int64(44)), sum(hits.ResolutionWidth + Int64(45)), sum(hits.ResolutionWidth + Int64(46)), sum(hits.ResolutionWidth + Int64(47)), sum(hits.ResolutionWidth + Int64(48)), sum(hits.ResolutionWidth + Int64(49)), sum(hits.ResolutionWidth + Int64(50)), sum(hits.ResolutionWidth + Int64(51)), sum(hits.ResolutionWidth + Int64(52)), sum(hits.ResolutionWidth + Int64(53)), sum(hits.ResolutionWidth + Int64(54)), sum(hits.ResolutionWidth + Int64(55)), sum(hits.ResolutionWidth + Int64(56)), sum(hits.ResolutionWidth + Int64(57)), sum(hits.ResolutionWidth + Int64(58)), sum(hits.ResolutionWidth + Int64(59)), sum(hits.ResolutionWidth + Int64(60)), sum(hits.ResolutionWidth + Int64(61)), sum(hits.ResolutionWidth + Int64(62)), sum(hits.ResolutionWidth + Int64(63)), sum(hits.ResolutionWidth + Int64(64)), sum(hits.ResolutionWidth + Int64(65)), sum(hits.ResolutionWidth + Int64(66)), sum(hits.ResolutionWidth + Int64(67)), sum(hits.ResolutionWidth + Int64(68)), sum(hits.ResolutionWidth + Int64(69)), sum(hits.ResolutionWidth + Int64(70)), sum(hits.ResolutionWidth + Int64(71)), sum(hits.ResolutionWidth + Int64(72)), sum(hits.ResolutionWidth + Int64(73)), sum(hits.ResolutionWidth + Int64(74)), sum(hits.ResolutionWidth + Int64(75)), sum(hits.ResolutionWidth + Int64(76)), sum(hits.ResolutionWidth + Int64(77)), sum(hits.ResolutionWidth + Int64(78)), sum(hits.ResolutionWidth + Int64(79)), sum(hits.ResolutionWidth + Int64(80)), sum(hits.ResolutionWidth + Int64(81)), sum(hits.ResolutionWidth + Int64(82)), sum(hits.ResolutionWidth + Int64(83)), sum(hits.ResolutionWidth + Int64(84)), sum(hits.ResolutionWidth + Int64(85)), sum(hits.ResolutionWidth + Int64(86)), sum(hits.ResolutionWidth + Int64(87)), sum(hits.ResolutionWidth + Int64(88)), sum(hits.ResolutionWidth + Int64(89))] @@ -660,7 +660,7 @@ mod tests { #[tokio::test] async fn test_clickbench_30() -> Result<()> { - let display = test_clickbench_query(30).await?; + let display = test_clickbench_query("q30").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [c@2 DESC], fetch=10 @@ -688,7 +688,7 @@ mod tests { #[tokio::test] #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (10 total):\n 8673025726158767406|1264438551|1|0|1990.00\n 5320052218057629211|-1703087277|1|0|1996.00\n 6244273852606083750|1554672832|1|0|1638.00\n 8628753750962053665|1215278356|1|0|1087.00\n 7035318163404387241|1326714320|1|0|1638.00\n 8431857775494210873|1237512945|1|0|1996.00\n 5110752526539992124|37611695|1|0|1917.00\n 8986794334343068049|1860752926|1|0|1638.00\n 8044147848299485837|1382122372|1|0|1368.00\n 7936057634954670727|1897481896|1|0|1638.00\n\nRows only in right (10 total):\n 5132615111782210132|-50313020|1|0|1368.00\n 5783789691451717551|-1310327384|1|0|1638.00\n 5756260993772351383|1484317883|1|0|375.00\n 7739310142000732364|991864113|1|0|1368.00\n 7593472904893539271|-151291403|1|0|1087.00\n 6339599967989898410|1543815587|1|0|1638.00\n 7794346560421945218|1645556180|1|0|1368.00\n 6112645108657361792|593586188|1|0|1638.00\n 6675910710751922756|-816379256|1|0|1368.00\n 5802727636196431835|1986422271|1|0|1996.00.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] async fn test_clickbench_31() -> Result<()> { - let display = test_clickbench_query(31).await?; + let display = test_clickbench_query("q31").await?; assert_snapshot!(display, @""); Ok(()) } @@ -696,14 +696,14 @@ mod tests { #[tokio::test] #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (10 total):\n 7643059318918524417|1767085700|1|0|0.00\n 5437163248266133938|-1465369615|1|0|0.00\n 9142541582422390102|-1465369615|1|0|0.00\n 8438994503411842126|-1465369615|1|0|0.00\n 7362096505818029859|-565678477|1|0|0.00\n 4928022308880516715|1699955284|1|0|0.00\n 5269769817689282522|1699955284|1|0|0.00\n 9081648050908046886|1699955284|1|0|0.00\n 6824181869275536503|1699955284|1|0|0.00\n 6905712404475757487|1552811156|1|0|0.00\n\nRows only in right (10 total):\n 6967277596165459879|-941091661|1|0|1368.00\n 5796237228224217668|-1310327384|1|0|1638.00\n 7218628137278606666|1511490240|1|0|1638.00\n 8314760197723815280|1566105210|1|0|1996.00\n 7053263954762394007|757778490|1|0|339.00\n 6283334114093174531|1216031795|1|0|1368.00\n 8818295356247036741|83042182|1|0|1638.00\n 6620528864937282562|-862894777|1|0|1996.00\n 8466121050002905379|83042182|1|0|1638.00\n 7554844936512227411|-1746904856|1|0|1368.00.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] async fn test_clickbench_32() -> Result<()> { - let display = test_clickbench_query(32).await?; + let display = test_clickbench_query("q32").await?; assert_snapshot!(display, @""); Ok(()) } #[tokio::test] async fn test_clickbench_33() -> Result<()> { - let display = test_clickbench_query(33).await?; + let display = test_clickbench_query("q33").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [c@1 DESC], fetch=10 @@ -728,7 +728,7 @@ mod tests { #[tokio::test] async fn test_clickbench_34() -> Result<()> { - let display = test_clickbench_query(34).await?; + let display = test_clickbench_query("q34").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [c@2 DESC], fetch=10 @@ -753,7 +753,7 @@ mod tests { #[tokio::test] async fn test_clickbench_35() -> Result<()> { - let display = test_clickbench_query(35).await?; + let display = test_clickbench_query("q35").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [c@4 DESC], fetch=10 @@ -779,7 +779,7 @@ mod tests { #[tokio::test] async fn test_clickbench_36() -> Result<()> { - let display = test_clickbench_query(36).await?; + let display = test_clickbench_query("q36").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [pageviews@1 DESC], fetch=10 @@ -806,7 +806,7 @@ mod tests { #[tokio::test] async fn test_clickbench_37() -> Result<()> { - let display = test_clickbench_query(37).await?; + let display = test_clickbench_query("q37").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [pageviews@1 DESC], fetch=10 @@ -833,7 +833,7 @@ mod tests { #[tokio::test] async fn test_clickbench_38() -> Result<()> { - let display = test_clickbench_query(38).await?; + let display = test_clickbench_query("q38").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ GlobalLimitExec: skip=1000, fetch=10 @@ -861,7 +861,7 @@ mod tests { #[tokio::test] async fn test_clickbench_39() -> Result<()> { - let display = test_clickbench_query(39).await?; + let display = test_clickbench_query("q39").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ GlobalLimitExec: skip=1000, fetch=10 @@ -889,7 +889,7 @@ mod tests { #[tokio::test] async fn test_clickbench_40() -> Result<()> { - let display = test_clickbench_query(40).await?; + let display = test_clickbench_query("q40").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ GlobalLimitExec: skip=100, fetch=10 @@ -917,7 +917,7 @@ mod tests { #[tokio::test] async fn test_clickbench_41() -> Result<()> { - let display = test_clickbench_query(41).await?; + let display = test_clickbench_query("q41").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ GlobalLimitExec: skip=10000, fetch=10 @@ -946,14 +946,14 @@ mod tests { #[tokio::test] #[ignore = "ordering mismatch: expected ordering: Some(LexOrdering { exprs: [PhysicalSortExpr { expr: ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }, options: SortOptions { descending: false, nulls_first: false } }], set: {ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }} }), actual ordering: Some(LexOrdering { exprs: [PhysicalSortExpr { expr: ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }, options: SortOptions { descending: false, nulls_first: false } }], set: {ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }} })"] async fn test_clickbench_42() -> Result<()> { - let display = test_clickbench_query(42).await?; + let display = test_clickbench_query("q42").await?; assert_snapshot!(display, @""); Ok(()) } static INIT_TEST_TPCDS_TABLES: OnceCell<()> = OnceCell::const_new(); - async fn test_clickbench_query(query_id: usize) -> Result { + async fn test_clickbench_query(query_id: &str) -> Result { let data_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(format!( "testdata/clickbench/plans_range{}-{}", FILE_RANGE.start, FILE_RANGE.end @@ -966,14 +966,14 @@ mod tests { }) .await; - let query_sql = clickbench::get_test_clickbench_query(query_id)?; + let query_sql = clickbench::get_query(query_id)?; let (d_ctx, _guard) = start_localhost_context(NUM_WORKERS, DefaultSessionBuilder).await; let d_ctx = d_ctx .with_distributed_files_per_task(FILES_PER_TASK)? .with_distributed_cardinality_effect_task_scale_factor(CARDINALITY_TASK_COUNT_FACTOR)?; - clickbench::register_tables(&d_ctx, &data_dir).await?; + benchmarks_common::register_tables(&d_ctx, &data_dir).await?; let df = d_ctx.sql(&query_sql).await?; let plan = df.create_physical_plan().await?; diff --git a/tests/tpcds_correctness_test.rs b/tests/tpcds_correctness_test.rs index 51fda085..9d5d2152 100644 --- a/tests/tpcds_correctness_test.rs +++ b/tests/tpcds_correctness_test.rs @@ -9,7 +9,7 @@ mod tests { use datafusion_distributed::test_utils::property_based::{ compare_ordering, compare_result_set, }; - use datafusion_distributed::test_utils::tpcds; + use datafusion_distributed::test_utils::{benchmarks_common, tpcds}; use datafusion_distributed::{ DefaultSessionBuilder, DistributedExec, DistributedExt, display_plan_ascii, }; @@ -26,359 +26,359 @@ mod tests { #[tokio::test] async fn test_tpcds_1() -> Result<()> { - test_tpcds_query(1).await + test_tpcds_query("q1").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_2() -> Result<()> { - test_tpcds_query(2).await + test_tpcds_query("q2").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_3() -> Result<()> { - test_tpcds_query(3).await + test_tpcds_query("q3").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_4() -> Result<()> { - test_tpcds_query(4).await + test_tpcds_query("q4").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_5() -> Result<()> { - test_tpcds_query(5).await + test_tpcds_query("q5").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_6() -> Result<()> { - test_tpcds_query(6).await + test_tpcds_query("q6").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_7() -> Result<()> { - test_tpcds_query(7).await + test_tpcds_query("q7").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_8() -> Result<()> { - test_tpcds_query(8).await + test_tpcds_query("q8").await } #[tokio::test] #[ignore = "expected no error but got: Arrow error: Invalid argument error: must either specify a row count or at least one column"] async fn test_tpcds_9() -> Result<()> { - test_tpcds_query(9).await + test_tpcds_query("q9").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_10() -> Result<()> { - test_tpcds_query(10).await + test_tpcds_query("q10").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_11() -> Result<()> { - test_tpcds_query(11).await + test_tpcds_query("q11").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_12() -> Result<()> { - test_tpcds_query(12).await + test_tpcds_query("q12").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_13() -> Result<()> { - test_tpcds_query(13).await + test_tpcds_query("q13").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_14() -> Result<()> { - test_tpcds_query(14).await + test_tpcds_query("q14").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_15() -> Result<()> { - test_tpcds_query(15).await + test_tpcds_query("q15").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_16() -> Result<()> { - test_tpcds_query(16).await + test_tpcds_query("q16").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_17() -> Result<()> { - test_tpcds_query(17).await + test_tpcds_query("q17").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_18() -> Result<()> { - test_tpcds_query(18).await + test_tpcds_query("q18").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_19() -> Result<()> { - test_tpcds_query(19).await + test_tpcds_query("q19").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_20() -> Result<()> { - test_tpcds_query(20).await + test_tpcds_query("q20").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_21() -> Result<()> { - test_tpcds_query(21).await + test_tpcds_query("q21").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_22() -> Result<()> { - test_tpcds_query(22).await + test_tpcds_query("q22").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_23() -> Result<()> { - test_tpcds_query(23).await + test_tpcds_query("q23").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_24() -> Result<()> { - test_tpcds_query(24).await + test_tpcds_query("q24").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_25() -> Result<()> { - test_tpcds_query(25).await + test_tpcds_query("q25").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_26() -> Result<()> { - test_tpcds_query(26).await + test_tpcds_query("q26").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_27() -> Result<()> { - test_tpcds_query(27).await + test_tpcds_query("q27").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_28() -> Result<()> { - test_tpcds_query(28).await + test_tpcds_query("q28").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_29() -> Result<()> { - test_tpcds_query(29).await + test_tpcds_query("q29").await } #[tokio::test] #[ignore = "Fails with column 'c_last_review_date_sk' not found"] async fn test_tpcds_30() -> Result<()> { - test_tpcds_query(30).await + test_tpcds_query("q30").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_31() -> Result<()> { - test_tpcds_query(31).await + test_tpcds_query("q31").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_32() -> Result<()> { - test_tpcds_query(32).await + test_tpcds_query("q32").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_33() -> Result<()> { - test_tpcds_query(33).await + test_tpcds_query("q33").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_34() -> Result<()> { - test_tpcds_query(34).await + test_tpcds_query("q34").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_35() -> Result<()> { - test_tpcds_query(35).await + test_tpcds_query("q35").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_36() -> Result<()> { - test_tpcds_query(36).await + test_tpcds_query("q36").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_37() -> Result<()> { - test_tpcds_query(37).await + test_tpcds_query("q37").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_38() -> Result<()> { - test_tpcds_query(38).await + test_tpcds_query("q38").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_39() -> Result<()> { - test_tpcds_query(39).await + test_tpcds_query("q39").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_40() -> Result<()> { - test_tpcds_query(40).await + test_tpcds_query("q40").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_41() -> Result<()> { - test_tpcds_query(41).await + test_tpcds_query("q41").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_42() -> Result<()> { - test_tpcds_query(42).await + test_tpcds_query("q42").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_43() -> Result<()> { - test_tpcds_query(43).await + test_tpcds_query("q43").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_44() -> Result<()> { - test_tpcds_query(44).await + test_tpcds_query("q44").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_45() -> Result<()> { - test_tpcds_query(45).await + test_tpcds_query("q45").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_46() -> Result<()> { - test_tpcds_query(46).await + test_tpcds_query("q46").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_47() -> Result<()> { - test_tpcds_query(47).await + test_tpcds_query("q47").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_48() -> Result<()> { - test_tpcds_query(48).await + test_tpcds_query("q48").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_49() -> Result<()> { - test_tpcds_query(49).await + test_tpcds_query("q49").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_50() -> Result<()> { - test_tpcds_query(50).await + test_tpcds_query("q50").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_51() -> Result<()> { - test_tpcds_query(51).await + test_tpcds_query("q51").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_52() -> Result<()> { - test_tpcds_query(52).await + test_tpcds_query("q52").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_53() -> Result<()> { - test_tpcds_query(53).await + test_tpcds_query("q53").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_54() -> Result<()> { - test_tpcds_query(54).await + test_tpcds_query("q54").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_55() -> Result<()> { - test_tpcds_query(55).await + test_tpcds_query("q55").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_56() -> Result<()> { - test_tpcds_query(56).await + test_tpcds_query("q56").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_57() -> Result<()> { - test_tpcds_query(57).await + test_tpcds_query("q57").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_58() -> Result<()> { - test_tpcds_query(58).await + test_tpcds_query("q58").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_59() -> Result<()> { - test_tpcds_query(59).await + test_tpcds_query("q59").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_60() -> Result<()> { - test_tpcds_query(60).await + test_tpcds_query("q60").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_61() -> Result<()> { - test_tpcds_query(61).await + test_tpcds_query("q61").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_62() -> Result<()> { - test_tpcds_query(62).await + test_tpcds_query("q62").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_63() -> Result<()> { - test_tpcds_query(63).await + test_tpcds_query("q63").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_64() -> Result<()> { - test_tpcds_query(64).await + test_tpcds_query("q64").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_65() -> Result<()> { - test_tpcds_query(65).await + test_tpcds_query("q65").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_66() -> Result<()> { - test_tpcds_query(66).await + test_tpcds_query("q66").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_67() -> Result<()> { - test_tpcds_query(67).await + test_tpcds_query("q67").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_68() -> Result<()> { - test_tpcds_query(68).await + test_tpcds_query("q68").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_69() -> Result<()> { - test_tpcds_query(69).await + test_tpcds_query("q69").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_70() -> Result<()> { - test_tpcds_query(70).await + test_tpcds_query("q70").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_71() -> Result<()> { - test_tpcds_query(71).await + test_tpcds_query("q71").await } #[tokio::test] @@ -387,137 +387,137 @@ mod tests { // long to execute that it's not worth the time. #[ignore = "Query takes too long to execute"] async fn test_tpcds_72() -> Result<()> { - test_tpcds_query(72).await + test_tpcds_query("q72").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_73() -> Result<()> { - test_tpcds_query(73).await + test_tpcds_query("q73").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_74() -> Result<()> { - test_tpcds_query(74).await + test_tpcds_query("q74").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_75() -> Result<()> { - test_tpcds_query(75).await + test_tpcds_query("q75").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_76() -> Result<()> { - test_tpcds_query(76).await + test_tpcds_query("q76").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_77() -> Result<()> { - test_tpcds_query(77).await + test_tpcds_query("q77").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_78() -> Result<()> { - test_tpcds_query(78).await + test_tpcds_query("q78").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_79() -> Result<()> { - test_tpcds_query(79).await + test_tpcds_query("q79").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_80() -> Result<()> { - test_tpcds_query(80).await + test_tpcds_query("q80").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_81() -> Result<()> { - test_tpcds_query(81).await + test_tpcds_query("q81").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_82() -> Result<()> { - test_tpcds_query(82).await + test_tpcds_query("q82").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_83() -> Result<()> { - test_tpcds_query(83).await + test_tpcds_query("q83").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_84() -> Result<()> { - test_tpcds_query(84).await + test_tpcds_query("q84").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_85() -> Result<()> { - test_tpcds_query(85).await + test_tpcds_query("q85").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_86() -> Result<()> { - test_tpcds_query(86).await + test_tpcds_query("q86").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_87() -> Result<()> { - test_tpcds_query(87).await + test_tpcds_query("q87").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_88() -> Result<()> { - test_tpcds_query(88).await + test_tpcds_query("q88").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_89() -> Result<()> { - test_tpcds_query(89).await + test_tpcds_query("q89").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_90() -> Result<()> { - test_tpcds_query(90).await + test_tpcds_query("q90").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_91() -> Result<()> { - test_tpcds_query(91).await + test_tpcds_query("q91").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_92() -> Result<()> { - test_tpcds_query(92).await + test_tpcds_query("q92").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_93() -> Result<()> { - test_tpcds_query(93).await + test_tpcds_query("q93").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_94() -> Result<()> { - test_tpcds_query(94).await + test_tpcds_query("q94").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_95() -> Result<()> { - test_tpcds_query(95).await + test_tpcds_query("q95").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_96() -> Result<()> { - test_tpcds_query(96).await + test_tpcds_query("q96").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_97() -> Result<()> { - test_tpcds_query(97).await + test_tpcds_query("q97").await } #[tokio::test(flavor = "multi_thread")] async fn test_tpcds_98() -> Result<()> { - test_tpcds_query(98).await + test_tpcds_query("q98").await } #[tokio::test] @@ -526,7 +526,7 @@ mod tests { // long to execute that it's not worth the time. #[ignore = "Query takes too long to execute"] async fn test_tpcds_99() -> Result<()> { - test_tpcds_query(99).await + test_tpcds_query("q99").await } static INIT_TEST_TPCDS_TABLES: OnceCell<()> = OnceCell::const_new(); @@ -541,21 +541,21 @@ mod tests { (plan.clone(), Arc::new(collect(plan, task_ctx).await)) // Collect execution errors, do not unwrap. } - async fn test_tpcds_query(query_id: usize) -> Result<()> { + async fn test_tpcds_query(query_id: &str) -> Result<()> { let data_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(format!( "testdata/tpcds/correctness_sf{SF}_partitions{PARQUET_PARTITIONS}" )); INIT_TEST_TPCDS_TABLES .get_or_init(|| async { if !fs::exists(&data_dir).unwrap_or(false) { - tpcds::generate_tpcds_data(&data_dir, SF, PARQUET_PARTITIONS) + tpcds::generate_data(&data_dir, SF, PARQUET_PARTITIONS) .await .unwrap(); } }) .await; - let query_sql = tpcds::get_test_tpcds_query(query_id)?; + let query_sql = tpcds::get_query(query_id)?; // Create a single node context to compare results to. let s_ctx = SessionContext::new(); @@ -565,8 +565,8 @@ mod tests { .with_distributed_files_per_task(FILES_PER_TASK)? .with_distributed_cardinality_effect_task_scale_factor(CARDINALITY_TASK_COUNT_FACTOR)?; - tpcds::register_tables(&s_ctx, &data_dir).await?; - tpcds::register_tables(&d_ctx, &data_dir).await?; + benchmarks_common::register_tables(&s_ctx, &data_dir).await?; + benchmarks_common::register_tables(&d_ctx, &data_dir).await?; let (s_plan, s_results) = run(&s_ctx, &query_sql).await; let (d_plan, d_results) = run(&d_ctx, &query_sql).await; diff --git a/tests/tpcds_plans_test.rs b/tests/tpcds_plans_test.rs index 5c80adac..adc6e710 100644 --- a/tests/tpcds_plans_test.rs +++ b/tests/tpcds_plans_test.rs @@ -2,7 +2,7 @@ mod tests { use datafusion::error::Result; use datafusion_distributed::test_utils::localhost::start_localhost_context; - use datafusion_distributed::test_utils::tpcds; + use datafusion_distributed::test_utils::{benchmarks_common, tpcds}; use datafusion_distributed::{ DefaultSessionBuilder, DistributedExec, DistributedExt, assert_snapshot, display_plan_ascii, }; @@ -19,7 +19,7 @@ mod tests { #[tokio::test] async fn test_tpcds_1() -> Result<()> { - let display = test_tpcds_query(1).await?; + let display = test_tpcds_query("q1").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [c_customer_id@0 ASC NULLS LAST], fetch=100 @@ -93,7 +93,7 @@ mod tests { #[tokio::test] async fn test_tpcds_2() -> Result<()> { - let display = test_tpcds_query(2).await?; + let display = test_tpcds_query("q2").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [d_week_seq1@0 ASC] @@ -196,7 +196,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_3() -> Result<()> { - let display = test_tpcds_query(3).await?; + let display = test_tpcds_query("q3").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [d_year@0 ASC NULLS LAST, sum_agg@3 DESC, brand_id@1 ASC NULLS LAST], fetch=100 @@ -232,7 +232,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_4() -> Result<()> { - let display = test_tpcds_query(4).await?; + let display = test_tpcds_query("q4").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [customer_id@0 ASC, customer_first_name@1 ASC, customer_last_name@2 ASC, customer_preferred_cust_flag@3 ASC], fetch=100 @@ -485,7 +485,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_5() -> Result<()> { - let display = test_tpcds_query(5).await?; + let display = test_tpcds_query("q5").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [channel@0 ASC, id@1 ASC], fetch=100 @@ -629,7 +629,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_6() -> Result<()> { - let display = test_tpcds_query(6).await?; + let display = test_tpcds_query("q6").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[state@0 as state, cnt@1 as cnt] @@ -716,7 +716,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_7() -> Result<()> { - let display = test_tpcds_query(7).await?; + let display = test_tpcds_query("q7").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST], fetch=100 @@ -780,7 +780,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_8() -> Result<()> { - let display = test_tpcds_query(8).await?; + let display = test_tpcds_query("q8").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [s_store_name@0 ASC NULLS LAST], fetch=100 @@ -855,7 +855,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_9() -> Result<()> { - let display = test_tpcds_query(9).await?; + let display = test_tpcds_query("q9").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ CoalescePartitionsExec @@ -1052,7 +1052,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_10() -> Result<()> { - let display = test_tpcds_query(10).await?; + let display = test_tpcds_query("q10").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [cd_gender@0 ASC NULLS LAST, cd_marital_status@1 ASC NULLS LAST, cd_education_status@2 ASC NULLS LAST, cd_purchase_estimate@4 ASC NULLS LAST, cd_credit_rating@6 ASC NULLS LAST, cd_dep_count@8 ASC NULLS LAST, cd_dep_employed_count@10 ASC NULLS LAST, cd_dep_college_count@12 ASC NULLS LAST], fetch=100 @@ -1139,7 +1139,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_11() -> Result<()> { - let display = test_tpcds_query(11).await?; + let display = test_tpcds_query("q11").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [customer_id@0 ASC, customer_first_name@1 ASC, customer_last_name@2 ASC, customer_preferred_cust_flag@3 ASC], fetch=100 @@ -1310,7 +1310,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_12() -> Result<()> { - let display = test_tpcds_query(12).await?; + let display = test_tpcds_query("q12").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [i_category@2 ASC NULLS LAST, i_class@3 ASC NULLS LAST, i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, revenueratio@6 ASC NULLS LAST], fetch=100 @@ -1355,7 +1355,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_13() -> Result<()> { - let display = test_tpcds_query(13).await?; + let display = test_tpcds_query("q13").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[avg(store_sales.ss_quantity)@0 as avg1, avg(store_sales.ss_ext_sales_price)@1 as avg2, avg(store_sales.ss_ext_wholesale_cost)@2 as avg3, sum(store_sales.ss_ext_wholesale_cost)@3 as sum(store_sales.ss_ext_wholesale_cost)] @@ -1414,7 +1414,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_14() -> Result<()> { - let display = test_tpcds_query(14).await?; + let display = test_tpcds_query("q14").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [channel@0 ASC, i_brand_id@1 ASC, i_class_id@2 ASC, i_category_id@3 ASC], fetch=100 @@ -2025,7 +2025,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_15() -> Result<()> { - let display = test_tpcds_query(15).await?; + let display = test_tpcds_query("q15").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [ca_zip@0 ASC], fetch=100 @@ -2087,7 +2087,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_16() -> Result<()> { - let display = test_tpcds_query(16).await?; + let display = test_tpcds_query("q16").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[count(alias1)@0 as order count, sum(alias2)@1 as total shipping cost, sum(alias3)@2 as total net profit] @@ -2150,7 +2150,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_17() -> Result<()> { - let display = test_tpcds_query(17).await?; + let display = test_tpcds_query("q17").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [i_item_id@0 ASC, i_item_desc@1 ASC, s_state@2 ASC], fetch=100 @@ -2249,7 +2249,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_18() -> Result<()> { - let display = test_tpcds_query(18).await?; + let display = test_tpcds_query("q18").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [ca_country@1 ASC, ca_state@2 ASC, ca_county@3 ASC, i_item_id@0 ASC], fetch=100 @@ -2339,7 +2339,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_19() -> Result<()> { - let display = test_tpcds_query(19).await?; + let display = test_tpcds_query("q19").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[brand_id@0 as brand_id, brand@1 as brand, i_manufact_id@2 as i_manufact_id, i_manufact@3 as i_manufact, ext_price@4 as ext_price] @@ -2416,7 +2416,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_20() -> Result<()> { - let display = test_tpcds_query(20).await?; + let display = test_tpcds_query("q20").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [i_category@2 ASC, i_class@3 ASC, i_item_id@0 ASC, i_item_desc@1 ASC, revenueratio@6 ASC], fetch=100 @@ -2461,7 +2461,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_21() -> Result<()> { - let display = test_tpcds_query(21).await?; + let display = test_tpcds_query("q21").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [w_warehouse_name@0 ASC, i_item_id@1 ASC], fetch=100 @@ -2510,7 +2510,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_22() -> Result<()> { - let display = test_tpcds_query(22).await?; + let display = test_tpcds_query("q22").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [qoh@4 ASC, i_product_name@0 ASC, i_brand@1 ASC, i_class@2 ASC, i_category@3 ASC], fetch=100 @@ -2557,7 +2557,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_23() -> Result<()> { - let display = test_tpcds_query(23).await?; + let display = test_tpcds_query("q23").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [c_last_name@0 ASC, c_first_name@1 ASC, sales@2 ASC], fetch=100 @@ -2851,7 +2851,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_24() -> Result<()> { - let display = test_tpcds_query(24).await?; + let display = test_tpcds_query("q24").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortExec: expr=[c_last_name@0 ASC NULLS LAST, c_first_name@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST], preserve_partitioning=[false] @@ -2975,7 +2975,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_25() -> Result<()> { - let display = test_tpcds_query(25).await?; + let display = test_tpcds_query("q25").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, s_store_id@2 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST], fetch=100 @@ -3061,7 +3061,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_26() -> Result<()> { - let display = test_tpcds_query(26).await?; + let display = test_tpcds_query("q26").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST], fetch=100 @@ -3125,7 +3125,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_27() -> Result<()> { - let display = test_tpcds_query(27).await?; + let display = test_tpcds_query("q27").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [i_item_id@0 ASC, s_state@1 ASC], fetch=100 @@ -3302,7 +3302,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_28() -> Result<()> { - let display = test_tpcds_query(28).await?; + let display = test_tpcds_query("q28").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[b1_lp@3 as b1_lp, b1_cnt@4 as b1_cnt, b1_cntd@5 as b1_cntd, b2_lp@6 as b2_lp, b2_cnt@7 as b2_cnt, b2_cntd@8 as b2_cntd, b3_lp@9 as b3_lp, b3_cnt@10 as b3_cnt, b3_cntd@11 as b3_cntd, b4_lp@12 as b4_lp, b4_cnt@13 as b4_cnt, b4_cntd@14 as b4_cntd, b5_lp@15 as b5_lp, b5_cnt@16 as b5_cnt, b5_cntd@17 as b5_cntd, b6_lp@0 as b6_lp, b6_cnt@1 as b6_cnt, b6_cntd@2 as b6_cntd] @@ -3387,7 +3387,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_29() -> Result<()> { - let display = test_tpcds_query(29).await?; + let display = test_tpcds_query("q29").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, s_store_id@2 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST], fetch=100 @@ -3478,13 +3478,13 @@ mod tests { #[tokio::test] #[ignore = "Fails with column 'c_last_review_date_sk' not found"] async fn test_tpcds_30() -> Result<()> { - let display = test_tpcds_query(30).await?; + let display = test_tpcds_query("q30").await?; assert_snapshot!(display, @r""); Ok(()) } #[tokio::test] async fn test_tpcds_31() -> Result<()> { - let display = test_tpcds_query(31).await?; + let display = test_tpcds_query("q31").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [ca_county@0 ASC NULLS LAST] @@ -3663,7 +3663,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_32() -> Result<()> { - let display = test_tpcds_query(32).await?; + let display = test_tpcds_query("q32").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[sum(catalog_sales.cs_ext_discount_amt)@0 as excess discount amount] @@ -3723,7 +3723,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_33() -> Result<()> { - let display = test_tpcds_query(33).await?; + let display = test_tpcds_query("q33").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [total_sales@1 ASC NULLS LAST], fetch=100 @@ -3851,7 +3851,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_34() -> Result<()> { - let display = test_tpcds_query(34).await?; + let display = test_tpcds_query("q34").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [c_last_name@0 ASC, c_first_name@1 ASC, c_salutation@2 ASC, c_preferred_cust_flag@3 DESC, ss_ticket_number@4 ASC] @@ -3913,7 +3913,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_35() -> Result<()> { - let display = test_tpcds_query(35).await?; + let display = test_tpcds_query("q35").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [ca_state@0 ASC, cd_gender@1 ASC, cd_marital_status@2 ASC, cd_dep_count@3 ASC, cd_dep_employed_count@8 ASC, cd_dep_college_count@13 ASC], fetch=100 @@ -4009,7 +4009,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_36() -> Result<()> { - let display = test_tpcds_query(36).await?; + let display = test_tpcds_query("q36").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [lochierarchy@3 DESC, CASE WHEN lochierarchy@3 = 0 THEN i_category@1 END ASC, rank_within_parent@4 ASC], fetch=100 @@ -4158,7 +4158,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_37() -> Result<()> { - let display = test_tpcds_query(37).await?; + let display = test_tpcds_query("q37").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST], fetch=100 @@ -4199,7 +4199,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_38() -> Result<()> { - let display = test_tpcds_query(38).await?; + let display = test_tpcds_query("q38").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] @@ -4296,7 +4296,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_39() -> Result<()> { - let display = test_tpcds_query(39).await?; + let display = test_tpcds_query("q39").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[wsk1@0 as wsk1, isk1@1 as isk1, dmoy1@2 as dmoy1, mean1@3 as mean1, cov1@4 as cov1, w_warehouse_sk@5 as w_warehouse_sk, i_item_sk@6 as i_item_sk, d_moy@7 as d_moy, mean@8 as mean, cov@9 as cov] @@ -4401,7 +4401,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_40() -> Result<()> { - let display = test_tpcds_query(40).await?; + let display = test_tpcds_query("q40").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [w_state@0 ASC NULLS LAST, i_item_id@1 ASC NULLS LAST], fetch=100 @@ -4460,7 +4460,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_41() -> Result<()> { - let display = test_tpcds_query(41).await?; + let display = test_tpcds_query("q41").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [i_product_name@0 ASC NULLS LAST], fetch=100 @@ -4499,7 +4499,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_42() -> Result<()> { - let display = test_tpcds_query(42).await?; + let display = test_tpcds_query("q42").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [sum(store_sales.ss_ext_sales_price)@3 DESC, d_year@0 ASC NULLS LAST, i_category_id@1 ASC NULLS LAST, i_category@2 ASC NULLS LAST], fetch=100 @@ -4534,7 +4534,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_43() -> Result<()> { - let display = test_tpcds_query(43).await?; + let display = test_tpcds_query("q43").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [s_store_name@0 ASC NULLS LAST, s_store_id@1 ASC NULLS LAST, sun_sales@2 ASC NULLS LAST, mon_sales@3 ASC NULLS LAST, tue_sales@4 ASC NULLS LAST, wed_sales@5 ASC NULLS LAST, thu_sales@6 ASC NULLS LAST, fri_sales@7 ASC NULLS LAST, sat_sales@8 ASC NULLS LAST], fetch=100 @@ -4576,7 +4576,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_44() -> Result<()> { - let display = test_tpcds_query(44).await?; + let display = test_tpcds_query("q44").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [rnk@0 ASC NULLS LAST], fetch=100 @@ -4699,7 +4699,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_45() -> Result<()> { - let display = test_tpcds_query(45).await?; + let display = test_tpcds_query("q45").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [ca_zip@0 ASC NULLS LAST, ca_city@1 ASC NULLS LAST], fetch=100 @@ -4775,7 +4775,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_46() -> Result<()> { - let display = test_tpcds_query(46).await?; + let display = test_tpcds_query("q46").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [c_last_name@0 ASC, c_first_name@1 ASC, ca_city@2 ASC, bought_city@3 ASC, ss_ticket_number@4 ASC], fetch=100 @@ -4846,7 +4846,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_47() -> Result<()> { - let display = test_tpcds_query(47).await?; + let display = test_tpcds_query("q47").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [sum_sales@7 - avg_monthly_sales@6 ASC NULLS LAST, i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_year@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST, avg_monthly_sales@6 ASC NULLS LAST, sum_sales@7 ASC NULLS LAST, psum@8 ASC NULLS LAST, nsum@9 ASC NULLS LAST], fetch=100 @@ -5018,7 +5018,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_48() -> Result<()> { - let display = test_tpcds_query(48).await?; + let display = test_tpcds_query("q48").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ AggregateExec: mode=Final, gby=[], aggr=[sum(store_sales.ss_quantity)] @@ -5065,7 +5065,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_49() -> Result<()> { - let display = test_tpcds_query(49).await?; + let display = test_tpcds_query("q49").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [channel@0 ASC, return_rank@3 ASC, currency_rank@4 ASC, item@1 ASC], fetch=100 @@ -5230,7 +5230,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_50() -> Result<()> { - let display = test_tpcds_query(50).await?; + let display = test_tpcds_query("q50").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [s_store_name@0 ASC NULLS LAST, s_company_id@1 ASC NULLS LAST, s_street_number@2 ASC NULLS LAST, s_street_name@3 ASC NULLS LAST, s_street_type@4 ASC NULLS LAST, s_suite_number@5 ASC NULLS LAST, s_city@6 ASC NULLS LAST, s_county@7 ASC NULLS LAST, s_state@8 ASC NULLS LAST, s_zip@9 ASC NULLS LAST], fetch=100 @@ -5288,7 +5288,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_51() -> Result<()> { - let display = test_tpcds_query(51).await?; + let display = test_tpcds_query("q51").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [item_sk@0 ASC, d_date@1 ASC], fetch=100 @@ -5360,7 +5360,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_52() -> Result<()> { - let display = test_tpcds_query(52).await?; + let display = test_tpcds_query("q52").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [d_year@0 ASC NULLS LAST, ext_price@3 DESC, brand_id@1 ASC NULLS LAST], fetch=100 @@ -5396,7 +5396,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_53() -> Result<()> { - let display = test_tpcds_query(53).await?; + let display = test_tpcds_query("q53").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [avg_quarterly_sales@2 ASC NULLS LAST, sum_sales@1 ASC NULLS LAST, i_manufact_id@0 ASC NULLS LAST], fetch=100 @@ -5453,7 +5453,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_54() -> Result<()> { - let display = test_tpcds_query(54).await?; + let display = test_tpcds_query("q54").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [segment@0 ASC, num_customers@1 ASC, segment_base@2 ASC NULLS LAST], fetch=100 @@ -5571,7 +5571,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_55() -> Result<()> { - let display = test_tpcds_query(55).await?; + let display = test_tpcds_query("q55").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[brand_id@0 as brand_id, brand@1 as brand, ext_price@2 as ext_price] @@ -5608,7 +5608,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_56() -> Result<()> { - let display = test_tpcds_query(56).await?; + let display = test_tpcds_query("q56").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [total_sales@1 ASC, i_item_id@0 ASC], fetch=100 @@ -5736,7 +5736,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_57() -> Result<()> { - let display = test_tpcds_query(57).await?; + let display = test_tpcds_query("q57").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [sum_sales@6 - avg_monthly_sales@5 ASC, i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_year@3 ASC NULLS LAST, d_moy@4 ASC NULLS LAST, avg_monthly_sales@5 ASC NULLS LAST, sum_sales@6 ASC NULLS LAST, psum@7 ASC NULLS LAST, nsum@8 ASC NULLS LAST], fetch=100 @@ -5908,7 +5908,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_58() -> Result<()> { - let display = test_tpcds_query(58).await?; + let display = test_tpcds_query("q58").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [item_id@0 ASC, ss_item_rev@1 ASC], fetch=100 @@ -6105,7 +6105,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_59() -> Result<()> { - let display = test_tpcds_query(59).await?; + let display = test_tpcds_query("q59").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [s_store_name1@0 ASC, s_store_id1@1 ASC, d_week_seq1@2 ASC], fetch=100 @@ -6210,7 +6210,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_60() -> Result<()> { - let display = test_tpcds_query(60).await?; + let display = test_tpcds_query("q60").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST, total_sales@1 ASC NULLS LAST], fetch=100 @@ -6338,7 +6338,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_61() -> Result<()> { - let display = test_tpcds_query(61).await?; + let display = test_tpcds_query("q61").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortExec: TopK(fetch=100), expr=[total@1 ASC NULLS LAST], preserve_partitioning=[false] @@ -6448,7 +6448,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_62() -> Result<()> { - let display = test_tpcds_query(62).await?; + let display = test_tpcds_query("q62").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [w_substr@0 ASC, sm_type@1 ASC, web_name@2 ASC], fetch=100 @@ -6494,7 +6494,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_63() -> Result<()> { - let display = test_tpcds_query(63).await?; + let display = test_tpcds_query("q63").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [i_manager_id@0 ASC NULLS LAST, avg_monthly_sales@2 ASC NULLS LAST, sum_sales@1 ASC NULLS LAST], fetch=100 @@ -6551,7 +6551,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_64() -> Result<()> { - let display = test_tpcds_query(64).await?; + let display = test_tpcds_query("q64").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[product_name@0 as product_name, store_name@1 as store_name, store_zip@2 as store_zip, b_street_number@3 as b_street_number, b_street_name@4 as b_street_name, b_city@5 as b_city, b_zip@6 as b_zip, c_street_number@7 as c_street_number, c_street_name@8 as c_street_name, c_city@9 as c_city, c_zip@10 as c_zip, cs1syear@11 as cs1syear, cs1cnt@12 as cs1cnt, s11@13 as s11, s21@14 as s21, s31@15 as s31, s12@16 as s12, s22@17 as s22, s32@18 as s32, syear@19 as syear, cnt@20 as cnt] @@ -6867,7 +6867,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_65() -> Result<()> { - let display = test_tpcds_query(65).await?; + let display = test_tpcds_query("q65").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [s_store_name@0 ASC, i_item_desc@1 ASC], fetch=100 @@ -6937,7 +6937,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_66() -> Result<()> { - let display = test_tpcds_query(66).await?; + let display = test_tpcds_query("q66").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [w_warehouse_name@0 ASC], fetch=100 @@ -7045,7 +7045,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_67() -> Result<()> { - let display = test_tpcds_query(67).await?; + let display = test_tpcds_query("q67").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [i_category@0 ASC, i_class@1 ASC, i_brand@2 ASC, i_product_name@3 ASC, d_year@4 ASC, d_qoy@5 ASC, d_moy@6 ASC, s_store_id@7 ASC, sumsales@8 ASC, rk@9 ASC], fetch=100 @@ -7097,7 +7097,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_68() -> Result<()> { - let display = test_tpcds_query(68).await?; + let display = test_tpcds_query("q68").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [c_last_name@0 ASC, ss_ticket_number@4 ASC], fetch=100 @@ -7168,7 +7168,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_69() -> Result<()> { - let display = test_tpcds_query(69).await?; + let display = test_tpcds_query("q69").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [cd_gender@0 ASC NULLS LAST, cd_marital_status@1 ASC NULLS LAST, cd_education_status@2 ASC NULLS LAST, cd_purchase_estimate@4 ASC NULLS LAST, cd_credit_rating@6 ASC NULLS LAST], fetch=100 @@ -7254,13 +7254,13 @@ mod tests { #[tokio::test] #[ignore = "The ordering of the column names in the first nodes is non deterministickI"] async fn test_tpcds_70() -> Result<()> { - let display = test_tpcds_query(70).await?; + let display = test_tpcds_query("q70").await?; assert_snapshot!(display, @r#""#); Ok(()) } #[tokio::test] async fn test_tpcds_71() -> Result<()> { - let display = test_tpcds_query(71).await?; + let display = test_tpcds_query("q71").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[brand_id@0 as brand_id, brand@1 as brand, t_hour@2 as t_hour, t_minute@3 as t_minute, ext_price@4 as ext_price] @@ -7344,7 +7344,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_72() -> Result<()> { - let display = test_tpcds_query(72).await?; + let display = test_tpcds_query("q72").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [total_cnt@5 DESC, i_item_desc@0 ASC, w_warehouse_name@1 ASC, d_week_seq@2 ASC], fetch=100 @@ -7457,7 +7457,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_73() -> Result<()> { - let display = test_tpcds_query(73).await?; + let display = test_tpcds_query("q73").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [cnt@5 DESC, c_last_name@0 ASC NULLS LAST] @@ -7519,7 +7519,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_74() -> Result<()> { - let display = test_tpcds_query(74).await?; + let display = test_tpcds_query("q74").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [customer_id@0 ASC], fetch=100 @@ -7690,7 +7690,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_75() -> Result<()> { - let display = test_tpcds_query(75).await?; + let display = test_tpcds_query("q75").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [sales_cnt_diff@8 ASC NULLS LAST, sales_amt_diff@9 ASC NULLS LAST], fetch=100 @@ -7925,7 +7925,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_76() -> Result<()> { - let display = test_tpcds_query(76).await?; + let display = test_tpcds_query("q76").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [channel@0 ASC, col_name@1 ASC, d_year@2 ASC, d_qoy@3 ASC, i_category@4 ASC], fetch=100 @@ -8062,7 +8062,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_77() -> Result<()> { - let display = test_tpcds_query(77).await?; + let display = test_tpcds_query("q77").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [channel@0 ASC, id@1 ASC, returns_@3 DESC], fetch=100 @@ -8245,7 +8245,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_78() -> Result<()> { - let display = test_tpcds_query(78).await?; + let display = test_tpcds_query("q78").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[ss_sold_year@0 as ss_sold_year, ss_item_sk@1 as ss_item_sk, ss_customer_sk@2 as ss_customer_sk, ratio@3 as ratio, store_qty@4 as store_qty, store_wholesale_cost@5 as store_wholesale_cost, store_sales_price@6 as store_sales_price, other_chan_qty@7 as other_chan_qty, other_chan_wholesale_cost@8 as other_chan_wholesale_cost, other_chan_sales_price@9 as other_chan_sales_price] @@ -8393,7 +8393,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_79() -> Result<()> { - let display = test_tpcds_query(79).await?; + let display = test_tpcds_query("q79").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [c_last_name@0 ASC, c_first_name@1 ASC, substr(ms.s_city,Int64(1),Int64(30))@2 ASC, profit@5 ASC, ss_ticket_number@3 ASC NULLS LAST], fetch=100 @@ -8454,7 +8454,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_80() -> Result<()> { - let display = test_tpcds_query(80).await?; + let display = test_tpcds_query("q80").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [channel@0 ASC, id@1 ASC], fetch=100 @@ -8674,7 +8674,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_81() -> Result<()> { - let display = test_tpcds_query(81).await?; + let display = test_tpcds_query("q81").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [c_customer_id@0 ASC NULLS LAST, c_salutation@1 ASC NULLS LAST, c_first_name@2 ASC NULLS LAST, c_last_name@3 ASC NULLS LAST, ca_street_number@4 ASC NULLS LAST, ca_street_name@5 ASC NULLS LAST, ca_street_type@6 ASC NULLS LAST, ca_suite_number@7 ASC NULLS LAST, ca_city@8 ASC NULLS LAST, ca_county@9 ASC NULLS LAST, ca_state@10 ASC NULLS LAST, ca_zip@11 ASC NULLS LAST, ca_country@12 ASC NULLS LAST, ca_gmt_offset@13 ASC NULLS LAST, ca_location_type@14 ASC NULLS LAST, ctr_total_return@15 ASC NULLS LAST], fetch=100 @@ -8758,7 +8758,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_82() -> Result<()> { - let display = test_tpcds_query(82).await?; + let display = test_tpcds_query("q82").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST], fetch=100 @@ -8799,7 +8799,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_83() -> Result<()> { - let display = test_tpcds_query(83).await?; + let display = test_tpcds_query("q83").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [item_id@0 ASC, sr_item_qty@1 ASC], fetch=100 @@ -8978,7 +8978,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_84() -> Result<()> { - let display = test_tpcds_query(84).await?; + let display = test_tpcds_query("q84").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[customer_id@0 as customer_id, customername@1 as customername] @@ -9035,7 +9035,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_85() -> Result<()> { - let display = test_tpcds_query(85).await?; + let display = test_tpcds_query("q85").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[substr(reason.r_reason_desc,Int64(1),Int64(20))@0 as substr(reason.r_reason_desc,Int64(1),Int64(20)), avg1@1 as avg1, avg2@2 as avg2, avg(web_returns.wr_fee)@3 as avg(web_returns.wr_fee)] @@ -9120,13 +9120,13 @@ mod tests { #[tokio::test] #[ignore = "The ordering of the column names in the first nodes is non deterministickI"] async fn test_tpcds_86() -> Result<()> { - let display = test_tpcds_query(86).await?; + let display = test_tpcds_query("q86").await?; assert_snapshot!(display, @r#""#); Ok(()) } #[tokio::test] async fn test_tpcds_87() -> Result<()> { - let display = test_tpcds_query(87).await?; + let display = test_tpcds_query("q87").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] @@ -9222,7 +9222,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_88() -> Result<()> { - let display = test_tpcds_query(88).await?; + let display = test_tpcds_query("q88").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[h8_30_to_9@1 as h8_30_to_9, h9_to_9_30@2 as h9_to_9_30, h9_30_to_10@3 as h9_30_to_10, h10_to_10_30@4 as h10_to_10_30, h10_30_to_11@5 as h10_30_to_11, h11_to_11_30@6 as h11_to_11_30, h11_30_to_12@7 as h11_30_to_12, h12_to_12_30@0 as h12_to_12_30] @@ -9580,7 +9580,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_89() -> Result<()> { - let display = test_tpcds_query(89).await?; + let display = test_tpcds_query("q89").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [sum_sales@6 - avg_monthly_sales@7 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST, i_category@0 ASC NULLS LAST, i_class@1 ASC NULLS LAST, i_brand@2 ASC NULLS LAST, s_company_name@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST, sum_sales@6 ASC NULLS LAST, avg_monthly_sales@7 ASC NULLS LAST], fetch=100 @@ -9637,7 +9637,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_90() -> Result<()> { - let display = test_tpcds_query(90).await?; + let display = test_tpcds_query("q90").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortExec: TopK(fetch=100), expr=[am_pm_ratio@0 ASC NULLS LAST], preserve_partitioning=[false] @@ -9733,7 +9733,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_91() -> Result<()> { - let display = test_tpcds_query(91).await?; + let display = test_tpcds_query("q91").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[call_center@0 as call_center, call_center_name@1 as call_center_name, manager@2 as manager, returns_loss@3 as returns_loss] @@ -9795,7 +9795,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_92() -> Result<()> { - let display = test_tpcds_query(92).await?; + let display = test_tpcds_query("q92").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[sum(web_sales.ws_ext_discount_amt)@0 as Excess Discount Amount] @@ -9855,7 +9855,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_93() -> Result<()> { - let display = test_tpcds_query(93).await?; + let display = test_tpcds_query("q93").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [sumsales@1 ASC, ss_customer_sk@0 ASC], fetch=100 @@ -9901,7 +9901,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_94() -> Result<()> { - let display = test_tpcds_query(94).await?; + let display = test_tpcds_query("q94").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[count(alias1)@0 as order count, sum(alias2)@1 as total shipping cost, sum(alias3)@2 as total net profit] @@ -9965,7 +9965,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_95() -> Result<()> { - let display = test_tpcds_query(95).await?; + let display = test_tpcds_query("q95").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[count(alias1)@0 as order count, sum(alias2)@1 as total shipping cost, sum(alias3)@2 as total net profit] @@ -10068,7 +10068,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_96() -> Result<()> { - let display = test_tpcds_query(96).await?; + let display = test_tpcds_query("q96").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] @@ -10120,7 +10120,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_97() -> Result<()> { - let display = test_tpcds_query(97).await?; + let display = test_tpcds_query("q97").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[sum(CASE WHEN ssci.customer_sk IS NOT NULL AND csci.customer_sk IS NULL THEN Int64(1) ELSE Int64(0) END)@0 as store_only, sum(CASE WHEN ssci.customer_sk IS NULL AND csci.customer_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END)@1 as catalog_only, sum(CASE WHEN ssci.customer_sk IS NOT NULL AND csci.customer_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END)@2 as store_and_catalog] @@ -10174,7 +10174,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_98() -> Result<()> { - let display = test_tpcds_query(98).await?; + let display = test_tpcds_query("q98").await?; assert_snapshot!(display, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [i_category@2 ASC, i_class@3 ASC, i_item_id@0 ASC, i_item_desc@1 ASC, revenueratio@6 ASC] @@ -10219,7 +10219,7 @@ mod tests { } #[tokio::test] async fn test_tpcds_99() -> Result<()> { - let display = test_tpcds_query(99).await?; + let display = test_tpcds_query("q99").await?; assert_snapshot!(display, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [w_substr@0 ASC, sm_type@1 ASC, cc_name_lower@2 ASC], fetch=100 @@ -10266,28 +10266,28 @@ mod tests { static INIT_TEST_TPCDS_TABLES: OnceCell<()> = OnceCell::const_new(); - async fn test_tpcds_query(query_id: usize) -> Result { + async fn test_tpcds_query(query_id: &str) -> Result { let data_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(format!( "testdata/tpcds/plans_sf{SF}_partitions{PARQUET_PARTITIONS}" )); INIT_TEST_TPCDS_TABLES .get_or_init(|| async { if !fs::exists(&data_dir).unwrap_or(false) { - tpcds::generate_tpcds_data(&data_dir, SF, PARQUET_PARTITIONS) + tpcds::generate_data(&data_dir, SF, PARQUET_PARTITIONS) .await .unwrap(); } }) .await; - let query_sql = tpcds::get_test_tpcds_query(query_id)?; + let query_sql = tpcds::get_query(query_id)?; // Make distributed localhost context to run queries let (d_ctx, _guard) = start_localhost_context(NUM_WORKERS, DefaultSessionBuilder).await; let d_ctx = d_ctx .with_distributed_files_per_task(FILES_PER_TASK)? .with_distributed_cardinality_effect_task_scale_factor(CARDINALITY_TASK_COUNT_FACTOR)?; - tpcds::register_tables(&d_ctx, &data_dir).await?; + benchmarks_common::register_tables(&d_ctx, &data_dir).await?; let df = d_ctx.sql(&query_sql).await?; let plan = df.create_physical_plan().await?; diff --git a/tests/tpch_correctness_test.rs b/tests/tpch_correctness_test.rs index 012dea32..d060aff0 100644 --- a/tests/tpch_correctness_test.rs +++ b/tests/tpch_correctness_test.rs @@ -4,7 +4,7 @@ mod tests { use datafusion::prelude::SessionContext; use datafusion_distributed::DefaultSessionBuilder; use datafusion_distributed::test_utils::localhost::start_localhost_context; - use datafusion_distributed::test_utils::tpch; + use datafusion_distributed::test_utils::{benchmarks_common, tpch}; use futures::TryStreamExt; use std::error::Error; use std::fmt::Display; @@ -18,52 +18,52 @@ mod tests { #[tokio::test] async fn test_tpch_1() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(1)?).await + test_tpch_query(tpch::get_query("q1")?).await } #[tokio::test] async fn test_tpch_2() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(2)?).await + test_tpch_query(tpch::get_query("q2")?).await } #[tokio::test] async fn test_tpch_3() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(3)?).await + test_tpch_query(tpch::get_query("q3")?).await } #[tokio::test] async fn test_tpch_4() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(4)?).await + test_tpch_query(tpch::get_query("q4")?).await } #[tokio::test] async fn test_tpch_5() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(5)?).await + test_tpch_query(tpch::get_query("q5")?).await } #[tokio::test] async fn test_tpch_6() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(6)?).await + test_tpch_query(tpch::get_query("q6")?).await } #[tokio::test] async fn test_tpch_7() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(7)?).await + test_tpch_query(tpch::get_query("q7")?).await } #[tokio::test] async fn test_tpch_8() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(8)?).await + test_tpch_query(tpch::get_query("q8")?).await } #[tokio::test] async fn test_tpch_9() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(9)?).await + test_tpch_query(tpch::get_query("q9")?).await } #[tokio::test] async fn test_tpch_10() -> Result<(), Box> { - let sql = tpch::get_test_tpch_query(10)?; + let sql = tpch::get_query("q10")?; // There is a chance that this query returns non-deterministic results if two entries // happen to have the exact same revenue. With small scales, this never happens, but with // bigger scales, this is very likely to happen. @@ -74,62 +74,62 @@ mod tests { #[tokio::test] async fn test_tpch_11() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(11)?).await + test_tpch_query(tpch::get_query("q11")?).await } #[tokio::test] async fn test_tpch_12() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(12)?).await + test_tpch_query(tpch::get_query("q12")?).await } #[tokio::test] async fn test_tpch_13() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(13)?).await + test_tpch_query(tpch::get_query("q13")?).await } #[tokio::test] async fn test_tpch_14() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(14)?).await + test_tpch_query(tpch::get_query("q14")?).await } #[tokio::test] async fn test_tpch_15() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(15)?).await + test_tpch_query(tpch::get_query("q15")?).await } #[tokio::test] async fn test_tpch_16() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(16)?).await + test_tpch_query(tpch::get_query("q16")?).await } #[tokio::test] async fn test_tpch_17() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(17)?).await + test_tpch_query(tpch::get_query("q17")?).await } #[tokio::test] async fn test_tpch_18() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(18)?).await + test_tpch_query(tpch::get_query("q18")?).await } #[tokio::test] async fn test_tpch_19() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(19)?).await + test_tpch_query(tpch::get_query("q19")?).await } #[tokio::test] async fn test_tpch_20() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(20)?).await + test_tpch_query(tpch::get_query("q20")?).await } #[tokio::test] async fn test_tpch_21() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(21)?).await + test_tpch_query(tpch::get_query("q21")?).await } #[tokio::test] async fn test_tpch_22() -> Result<(), Box> { - test_tpch_query(tpch::get_test_tpch_query(22)?).await + test_tpch_query(tpch::get_query("q22")?).await } // test_tpch_query runs each TPC-H query twice - once in a distributed manner and once @@ -156,18 +156,7 @@ mod tests { .execution .target_partitions = PARTITIONS; - // Register tables for first context - for table_name in [ - "lineitem", "orders", "part", "partsupp", "customer", "nation", "region", "supplier", - ] { - let query_path = data_dir.join(table_name); - ctx.register_parquet( - table_name, - query_path.to_string_lossy().as_ref(), - datafusion::prelude::ParquetReadOptions::default(), - ) - .await?; - } + benchmarks_common::register_tables(&ctx, &data_dir).await?; // Query 15 has three queries in it, one creating the view, the second // executing, which we want to capture the output of, and the third diff --git a/tests/tpch_explain_analyze.rs b/tests/tpch_explain_analyze.rs index b03947f1..9430dfc1 100644 --- a/tests/tpch_explain_analyze.rs +++ b/tests/tpch_explain_analyze.rs @@ -3,7 +3,7 @@ mod tests { use datafusion::physical_plan::execute_stream; use datafusion::prelude::SessionContext; use datafusion_distributed::test_utils::localhost::start_localhost_context; - use datafusion_distributed::test_utils::tpch; + use datafusion_distributed::test_utils::{benchmarks_common, tpch}; use datafusion_distributed::{ DefaultSessionBuilder, DistributedExt, assert_snapshot, explain_analyze, }; @@ -20,7 +20,7 @@ mod tests { #[tokio::test] async fn test_tpch_1() -> Result<(), Box> { - let plan = test_tpch_query(1).await?; + let plan = test_tpch_query("q1").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST], metrics=[output_rows=4, elapsed_compute=, output_bytes=] @@ -48,7 +48,7 @@ mod tests { #[tokio::test] async fn test_tpch_2() -> Result<(), Box> { - let plan = test_tpch_query(2).await?; + let plan = test_tpch_query("q2").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], metrics=[output_rows=11264, elapsed_compute=, output_bytes=B] @@ -124,7 +124,7 @@ mod tests { #[tokio::test] async fn test_tpch_3() -> Result<(), Box> { - let plan = test_tpch_query(3).await?; + let plan = test_tpch_query("q3").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [revenue@1 DESC, o_orderdate@2 ASC NULLS LAST], metrics=[output_rows=1216, elapsed_compute=, output_bytes=B] @@ -160,7 +160,7 @@ mod tests { #[tokio::test] async fn test_tpch_4() -> Result<(), Box> { - let plan = test_tpch_query(4).await?; + let plan = test_tpch_query("q4").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [o_orderpriority@0 ASC NULLS LAST], metrics=[output_rows=5, elapsed_compute=, output_bytes=B] @@ -190,7 +190,7 @@ mod tests { #[tokio::test] async fn test_tpch_5() -> Result<(), Box> { - let plan = test_tpch_query(5).await?; + let plan = test_tpch_query("q5").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [revenue@1 DESC], metrics=[output_rows=5, elapsed_compute=, output_bytes=] @@ -243,7 +243,7 @@ mod tests { #[tokio::test] async fn test_tpch_6() -> Result<(), Box> { - let plan = test_tpch_query(6).await?; + let plan = test_tpch_query("q6").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[sum(lineitem.l_extendedprice * lineitem.l_discount)@0 as revenue], metrics=[output_rows=1, elapsed_compute=, output_bytes=] @@ -264,7 +264,7 @@ mod tests { #[tokio::test] async fn test_tpch_7() -> Result<(), Box> { - let plan = test_tpch_query(7).await?; + let plan = test_tpch_query("q7").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST], metrics=[output_rows=4, elapsed_compute=, output_bytes=] @@ -339,7 +339,7 @@ mod tests { #[tokio::test] async fn test_tpch_8() -> Result<(), Box> { - let plan = test_tpch_query(8).await?; + let plan = test_tpch_query("q8").await?; assert_snapshot!(plan, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [o_year@0 ASC NULLS LAST], metrics=[output_rows=2, elapsed_compute=, output_bytes=] @@ -424,7 +424,7 @@ mod tests { #[tokio::test] async fn test_tpch_9() -> Result<(), Box> { - let plan = test_tpch_query(9).await?; + let plan = test_tpch_query("q9").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [nation@0 ASC NULLS LAST, o_year@1 DESC], metrics=[output_rows=175, elapsed_compute=, output_bytes=B] @@ -491,7 +491,7 @@ mod tests { #[tokio::test] async fn test_tpch_10() -> Result<(), Box> { - let plan = test_tpch_query(10).await?; + let plan = test_tpch_query("q10").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [revenue@2 DESC], metrics=[output_rows=3767, elapsed_compute=, output_bytes=B] @@ -531,7 +531,7 @@ mod tests { #[tokio::test] async fn test_tpch_11() -> Result<(), Box> { - let plan = test_tpch_query(11).await?; + let plan = test_tpch_query("q11").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [value@1 DESC], metrics=[output_rows=2541, elapsed_compute=, output_bytes=B] @@ -586,7 +586,7 @@ mod tests { #[tokio::test] async fn test_tpch_12() -> Result<(), Box> { - let plan = test_tpch_query(12).await?; + let plan = test_tpch_query("q12").await?; assert_snapshot!(plan, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [l_shipmode@0 ASC NULLS LAST], metrics=[output_rows=2, elapsed_compute=, output_bytes=] @@ -627,7 +627,7 @@ mod tests { #[tokio::test] async fn test_tpch_13() -> Result<(), Box> { - let plan = test_tpch_query(13).await?; + let plan = test_tpch_query("q13").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [custdist@1 DESC, c_count@0 DESC], metrics=[output_rows=37, elapsed_compute=, output_bytes=] @@ -661,7 +661,7 @@ mod tests { #[tokio::test] async fn test_tpch_14() -> Result<(), Box> { - let plan = test_tpch_query(14).await?; + let plan = test_tpch_query("q14").await?; assert_snapshot!(plan, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[100 * CAST(sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END)@0 AS Float64) / CAST(sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 AS Float64) as promo_revenue], metrics=[output_rows=1, elapsed_compute=, output_bytes=] @@ -697,7 +697,7 @@ mod tests { #[tokio::test] async fn test_tpch_15() -> Result<(), Box> { - let plan = test_tpch_query(15).await?; + let plan = test_tpch_query("q15").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [s_suppkey@0 ASC NULLS LAST], metrics=[output_rows=1, elapsed_compute=, output_bytes=B] @@ -745,7 +745,7 @@ mod tests { #[tokio::test] async fn test_tpch_16() -> Result<(), Box> { - let plan = test_tpch_query(16).await?; + let plan = test_tpch_query("q16").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], metrics=[output_rows=2762, elapsed_compute=, output_bytes=B] @@ -788,7 +788,7 @@ mod tests { #[tokio::test] async fn test_tpch_17() -> Result<(), Box> { - let plan = test_tpch_query(17).await?; + let plan = test_tpch_query("q17").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[CAST(sum(lineitem.l_extendedprice)@0 AS Float64) / 7 as avg_yearly], metrics=[output_rows=1, elapsed_compute=, output_bytes=] @@ -834,7 +834,7 @@ mod tests { #[tokio::test] async fn test_tpch_18() -> Result<(), Box> { - let plan = test_tpch_query(18).await?; + let plan = test_tpch_query("q18").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST], metrics=[output_rows=5, elapsed_compute=, output_bytes=] @@ -897,7 +897,7 @@ mod tests { #[tokio::test] async fn test_tpch_19() -> Result<(), Box> { - let plan = test_tpch_query(19).await?; + let plan = test_tpch_query("q19").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@0 as revenue], metrics=[output_rows=1, elapsed_compute=, output_bytes=] @@ -924,7 +924,7 @@ mod tests { #[tokio::test] async fn test_tpch_20() -> Result<(), Box> { - let plan = test_tpch_query(20).await?; + let plan = test_tpch_query("q20").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [s_name@0 ASC NULLS LAST], metrics=[output_rows=144, elapsed_compute=, output_bytes=B] @@ -976,7 +976,7 @@ mod tests { #[tokio::test] async fn test_tpch_21() -> Result<(), Box> { - let plan = test_tpch_query(21).await?; + let plan = test_tpch_query("q21").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [numwait@1 DESC, s_name@0 ASC NULLS LAST], metrics=[output_rows=47, elapsed_compute=, output_bytes=B] @@ -1030,7 +1030,7 @@ mod tests { #[tokio::test] async fn test_tpch_22() -> Result<(), Box> { - let plan = test_tpch_query(22).await?; + let plan = test_tpch_query("q22").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [cntrycode@0 ASC NULLS LAST], metrics=[output_rows=7, elapsed_compute=, output_bytes=] @@ -1071,15 +1071,12 @@ mod tests { // test_tpch_query runs each TPC-H query in a distributed manner while collecting metrics. // This allows us to call `explain_analyze` on the resulting executed plan. - async fn test_tpch_query(query_id: u8) -> Result> { + async fn test_tpch_query(query_id: &str) -> Result> { let (mut ctx, _guard) = start_localhost_context(4, DefaultSessionBuilder).await; ctx.set_distributed_metrics_collection(true)?; - run_tpch_query(ctx, query_id).await - } - async fn run_tpch_query(ctx: SessionContext, query_id: u8) -> Result> { let data_dir = ensure_tpch_data(TPCH_SCALE_FACTOR, TPCH_DATA_PARTS).await; - let sql = tpch::get_test_tpch_query(query_id)?; + let sql = tpch::get_query(query_id)?; ctx.state_ref() .write() .config_mut() @@ -1087,23 +1084,12 @@ mod tests { .execution .target_partitions = PARTITIONS; - // Register tables for first context - for table_name in [ - "lineitem", "orders", "part", "partsupp", "customer", "nation", "region", "supplier", - ] { - let query_path = data_dir.join(table_name); - ctx.register_parquet( - table_name, - query_path.to_string_lossy().as_ref(), - datafusion::prelude::ParquetReadOptions::default(), - ) - .await?; - } + benchmarks_common::register_tables(&ctx, &data_dir).await?; // Query 15 has three queries in it, one creating the view, the second // executing, which we want to capture the output of, and the third // tearing down the view - let plan = if query_id == 15 { + let plan = if query_id == "q15" { let queries: Vec<&str> = sql .split(';') .map(str::trim) diff --git a/tests/tpch_plans_test.rs b/tests/tpch_plans_test.rs index 9a8a7a08..8d246d9f 100644 --- a/tests/tpch_plans_test.rs +++ b/tests/tpch_plans_test.rs @@ -2,7 +2,7 @@ mod tests { use datafusion::prelude::SessionContext; use datafusion_distributed::test_utils::localhost::start_localhost_context; - use datafusion_distributed::test_utils::tpch; + use datafusion_distributed::test_utils::{benchmarks_common, tpch}; use datafusion_distributed::{DefaultSessionBuilder, assert_snapshot, display_plan_ascii}; use std::error::Error; use std::fs; @@ -15,7 +15,7 @@ mod tests { #[tokio::test] async fn test_tpch_1() -> Result<(), Box> { - let plan = test_tpch_query(1).await?; + let plan = test_tpch_query("q1").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST] @@ -43,7 +43,7 @@ mod tests { #[tokio::test] async fn test_tpch_2() -> Result<(), Box> { - let plan = test_tpch_query(2).await?; + let plan = test_tpch_query("q2").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST] @@ -119,7 +119,7 @@ mod tests { #[tokio::test] async fn test_tpch_3() -> Result<(), Box> { - let plan = test_tpch_query(3).await?; + let plan = test_tpch_query("q3").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [revenue@1 DESC, o_orderdate@2 ASC NULLS LAST] @@ -155,7 +155,7 @@ mod tests { #[tokio::test] async fn test_tpch_4() -> Result<(), Box> { - let plan = test_tpch_query(4).await?; + let plan = test_tpch_query("q4").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [o_orderpriority@0 ASC NULLS LAST] @@ -185,7 +185,7 @@ mod tests { #[tokio::test] async fn test_tpch_5() -> Result<(), Box> { - let plan = test_tpch_query(5).await?; + let plan = test_tpch_query("q5").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [revenue@1 DESC] @@ -238,7 +238,7 @@ mod tests { #[tokio::test] async fn test_tpch_6() -> Result<(), Box> { - let plan = test_tpch_query(6).await?; + let plan = test_tpch_query("q6").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[sum(lineitem.l_extendedprice * lineitem.l_discount)@0 as revenue] @@ -259,7 +259,7 @@ mod tests { #[tokio::test] async fn test_tpch_7() -> Result<(), Box> { - let plan = test_tpch_query(7).await?; + let plan = test_tpch_query("q7").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST] @@ -321,7 +321,7 @@ mod tests { #[tokio::test] async fn test_tpch_8() -> Result<(), Box> { - let plan = test_tpch_query(8).await?; + let plan = test_tpch_query("q8").await?; assert_snapshot!(plan, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [o_year@0 ASC NULLS LAST] @@ -390,7 +390,7 @@ mod tests { #[tokio::test] async fn test_tpch_9() -> Result<(), Box> { - let plan = test_tpch_query(9).await?; + let plan = test_tpch_query("q9").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [nation@0 ASC NULLS LAST, o_year@1 DESC] @@ -457,7 +457,7 @@ mod tests { #[tokio::test] async fn test_tpch_10() -> Result<(), Box> { - let plan = test_tpch_query(10).await?; + let plan = test_tpch_query("q10").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [revenue@2 DESC] @@ -497,7 +497,7 @@ mod tests { #[tokio::test] async fn test_tpch_11() -> Result<(), Box> { - let plan = test_tpch_query(11).await?; + let plan = test_tpch_query("q11").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [value@1 DESC] @@ -552,7 +552,7 @@ mod tests { #[tokio::test] async fn test_tpch_12() -> Result<(), Box> { - let plan = test_tpch_query(12).await?; + let plan = test_tpch_query("q12").await?; assert_snapshot!(plan, @r#" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [l_shipmode@0 ASC NULLS LAST] @@ -593,7 +593,7 @@ mod tests { #[tokio::test] async fn test_tpch_13() -> Result<(), Box> { - let plan = test_tpch_query(13).await?; + let plan = test_tpch_query("q13").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [custdist@1 DESC, c_count@0 DESC] @@ -627,7 +627,7 @@ mod tests { #[tokio::test] async fn test_tpch_14() -> Result<(), Box> { - let plan = test_tpch_query(14).await?; + let plan = test_tpch_query("q14").await?; assert_snapshot!(plan, @r#" ProjectionExec: expr=[100 * CAST(sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END)@0 AS Float64) / CAST(sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 AS Float64) as promo_revenue] AggregateExec: mode=Final, gby=[], aggr=[sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] @@ -647,7 +647,7 @@ mod tests { #[tokio::test] async fn test_tpch_15() -> Result<(), Box> { - let plan = test_tpch_query(15).await?; + let plan = test_tpch_query("q15").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [s_suppkey@0 ASC NULLS LAST] @@ -695,7 +695,7 @@ mod tests { #[tokio::test] async fn test_tpch_16() -> Result<(), Box> { - let plan = test_tpch_query(16).await?; + let plan = test_tpch_query("q16").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST] @@ -738,7 +738,7 @@ mod tests { #[tokio::test] async fn test_tpch_17() -> Result<(), Box> { - let plan = test_tpch_query(17).await?; + let plan = test_tpch_query("q17").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[CAST(sum(lineitem.l_extendedprice)@0 AS Float64) / 7 as avg_yearly] @@ -777,7 +777,7 @@ mod tests { #[tokio::test] async fn test_tpch_18() -> Result<(), Box> { - let plan = test_tpch_query(18).await?; + let plan = test_tpch_query("q18").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST] @@ -819,7 +819,7 @@ mod tests { #[tokio::test] async fn test_tpch_19() -> Result<(), Box> { - let plan = test_tpch_query(19).await?; + let plan = test_tpch_query("q19").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@0 as revenue] @@ -846,7 +846,7 @@ mod tests { #[tokio::test] async fn test_tpch_20() -> Result<(), Box> { - let plan = test_tpch_query(20).await?; + let plan = test_tpch_query("q20").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [s_name@0 ASC NULLS LAST] @@ -898,7 +898,7 @@ mod tests { #[tokio::test] async fn test_tpch_21() -> Result<(), Box> { - let plan = test_tpch_query(21).await?; + let plan = test_tpch_query("q21").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [numwait@1 DESC, s_name@0 ASC NULLS LAST] @@ -952,7 +952,7 @@ mod tests { #[tokio::test] async fn test_tpch_22() -> Result<(), Box> { - let plan = test_tpch_query(22).await?; + let plan = test_tpch_query("q22").await?; assert_snapshot!(plan, @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ SortPreservingMergeExec: [cntrycode@0 ASC NULLS LAST] @@ -992,14 +992,10 @@ mod tests { } // test_tpch_query generates and displays a distributed plan for each TPC-H query. - async fn test_tpch_query(query_id: u8) -> Result> { + async fn test_tpch_query(query_id: &str) -> Result> { let (ctx, _guard) = start_localhost_context(4, DefaultSessionBuilder).await; - run_tpch_query(ctx, query_id).await - } - - async fn run_tpch_query(ctx: SessionContext, query_id: u8) -> Result> { let data_dir = ensure_tpch_data(TPCH_SCALE_FACTOR, TPCH_DATA_PARTS).await; - let sql = tpch::get_test_tpch_query(query_id)?; + let sql = tpch::get_query(query_id)?; ctx.state_ref() .write() .config_mut() @@ -1007,23 +1003,12 @@ mod tests { .execution .target_partitions = PARTITIONS; - // Register tables for first context - for table_name in [ - "lineitem", "orders", "part", "partsupp", "customer", "nation", "region", "supplier", - ] { - let query_path = data_dir.join(table_name); - ctx.register_parquet( - table_name, - query_path.to_string_lossy().as_ref(), - datafusion::prelude::ParquetReadOptions::default(), - ) - .await?; - } + benchmarks_common::register_tables(&ctx, &data_dir).await?; // Query 15 has three queries in it, one creating the view, the second // executing, which we want to capture the output of, and the third // tearing down the view - let plan = if query_id == 15 { + let plan = if query_id == "q15" { let queries: Vec<&str> = sql .split(';') .map(str::trim) From d4212bb1311c2f603575faf256f15d4532d2479e Mon Sep 17 00:00:00 2001 From: Gabriel <45515538+gabotechs@users.noreply.github.com> Date: Wed, 14 Jan 2026 08:22:25 +0100 Subject: [PATCH 19/27] Worker network stream optimizations (#283) * Allow passing any query id for benchmarks * Allow passing a custom query * Tidy up benchmarking code * Allow passing a custom query * Introduce worker_connection_pool.rs for handing inter-worker connection batching * Allow passing any query id for benchmarks * Allow passing a custom query * Tidy up benchmarking code * Use ranges for the worker connection pool * Better debug messages --- src/common/map_last_stream.rs | 25 +- src/execution_plans/common.rs | 122 -------- src/execution_plans/network_coalesce.rs | 122 +++----- src/execution_plans/network_shuffle.rs | 113 ++------ src/flight_service/do_get.rs | 183 ++++++------ src/flight_service/mod.rs | 5 +- src/flight_service/spawn_select_all.rs | 109 +++++++ src/flight_service/worker_connection_pool.rs | 282 +++++++++++++++++++ src/metrics/metrics_collecting_stream.rs | 257 ----------------- src/metrics/mod.rs | 2 - src/protobuf/app_metadata.rs | 19 +- src/protobuf/distributed_codec.rs | 3 + src/protobuf/mod.rs | 5 +- 13 files changed, 586 insertions(+), 661 deletions(-) create mode 100644 src/flight_service/spawn_select_all.rs create mode 100644 src/flight_service/worker_connection_pool.rs delete mode 100644 src/metrics/metrics_collecting_stream.rs diff --git a/src/common/map_last_stream.rs b/src/common/map_last_stream.rs index d0eb7798..0017520e 100644 --- a/src/common/map_last_stream.rs +++ b/src/common/map_last_stream.rs @@ -1,13 +1,12 @@ use futures::{Stream, StreamExt, stream}; use std::task::Poll; -/// Maps the last element of the provided stream. +/// Maps each element of the provided stream, providing an additional flag that determines +/// whether the mapped element is the last one or not. pub(crate) fn map_last_stream( mut input: impl Stream + Unpin, - map_f: impl FnOnce(T) -> T, + map_f: impl Fn(T, bool) -> T, ) -> impl Stream + Unpin { - let mut final_closure = Some(map_f); - // this is used to peek the new value so that we can map upon emitting the last message let mut current_value = None; @@ -23,8 +22,7 @@ pub(crate) fn map_last_stream( Some(existing) => { current_value = Some(new_val); - - Poll::Ready(Some(existing)) + Poll::Ready(Some(map_f(existing, false))) } } } @@ -33,12 +31,7 @@ pub(crate) fn map_last_stream( Some(existing) => { // make sure we wake ourselves to finish the stream cx.waker().wake_by_ref(); - - if let Some(closure) = final_closure.take() { - Poll::Ready(Some(closure(existing))) - } else { - unreachable!("the closure is only executed once") - } + Poll::Ready(Some(map_f(existing, true))) } None => Poll::Ready(None), }, @@ -53,7 +46,7 @@ mod tests { #[tokio::test] async fn test_map_last_stream_empty_stream() { let input = stream::empty::(); - let mapped = map_last_stream(input, |x| x + 10); + let mapped = map_last_stream(input, |x, _| x + 10); let result: Vec = mapped.collect().await; assert_eq!(result, Vec::::new()); } @@ -61,7 +54,7 @@ mod tests { #[tokio::test] async fn test_map_last_stream_single_element() { let input = stream::iter(vec![5]); - let mapped = map_last_stream(input, |x| x * 2); + let mapped = map_last_stream(input, |x, _| x * 2); let result: Vec = mapped.collect().await; assert_eq!(result, vec![10]); } @@ -69,7 +62,7 @@ mod tests { #[tokio::test] async fn test_map_last_stream_multiple_elements() { let input = stream::iter(vec![1, 2, 3, 4]); - let mapped = map_last_stream(input, |x| x + 100); + let mapped = map_last_stream(input, |x, is_last| if is_last { x + 100 } else { x }); let result: Vec = mapped.collect().await; assert_eq!(result, vec![1, 2, 3, 104]); // Only the last element is transformed } @@ -77,7 +70,7 @@ mod tests { #[tokio::test] async fn test_map_last_stream_preserves_order() { let input = stream::iter(vec![10, 20, 30, 40, 50]); - let mapped = map_last_stream(input, |x| x - 50); + let mapped = map_last_stream(input, |x, is_last| if is_last { x - 50 } else { x }); let result: Vec = mapped.collect().await; assert_eq!(result, vec![10, 20, 30, 40, 0]); // Last element: 50 - 50 = 0 } diff --git a/src/execution_plans/common.rs b/src/execution_plans/common.rs index ffccc375..a1557a23 100644 --- a/src/execution_plans/common.rs +++ b/src/execution_plans/common.rs @@ -1,13 +1,5 @@ -use crate::DistributedConfig; -use datafusion::arrow::array::RecordBatch; -use datafusion::common::runtime::SpawnedTask; -use datafusion::execution::memory_pool::{MemoryConsumer, MemoryPool}; use datafusion::physical_expr::Partitioning; use datafusion::physical_plan::PlanProperties; -use futures::{Stream, StreamExt}; -use http::HeaderMap; -use std::sync::Arc; -use tokio_stream::wrappers::UnboundedReceiverStream; pub(super) fn scale_partitioning_props( props: &PlanProperties, @@ -31,117 +23,3 @@ pub(super) fn scale_partitioning( Partitioning::UnknownPartitioning(p) => Partitioning::UnknownPartitioning(f(*p)), } } - -/// Manual propagation of the [DistributedConfig] fields relevant for execution. Can be removed -/// after https://github.com/datafusion-contrib/datafusion-distributed/issues/247 is fixed, as this will become automatic. -pub(super) fn manually_propagate_distributed_config( - mut headers: HeaderMap, - d_cfg: &DistributedConfig, -) -> HeaderMap { - headers.insert( - "distributed.collect_metrics", - d_cfg.collect_metrics.to_string().parse().unwrap(), - ); - headers -} - -/// Consumes all the provided streams in parallel sending their produced messages to a single -/// queue in random order. The resulting queue is returned as a stream. -// FIXME: It should not be necessary to do this, it should be fine to just consume -// all the messages with a normal tokio::stream::select_all, however, that has the chance -// of deadlocking the stream on the server side (https://github.com/datafusion-contrib/datafusion-distributed/issues/228). -// Even having these channels bounded would result in deadlocks (learned it the hard way). -// Until we figure out what's wrong there, this is a good enough solution. -pub(super) fn spawn_select_all( - inner: Vec, - pool: Arc, -) -> impl Stream> -where - T: Stream> + Send + Unpin + 'static, - El: MemoryFootPrint + Send + 'static, - Err: Send + 'static, -{ - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - - let mut tasks = vec![]; - for mut t in inner { - let tx = tx.clone(); - let pool = Arc::clone(&pool); - let consumer = MemoryConsumer::new("NetworkBoundary"); - - tasks.push(SpawnedTask::spawn(async move { - while let Some(msg) = t.next().await { - let mut reservation = consumer.clone_with_new_id().register(&pool); - if let Ok(msg) = &msg { - reservation.grow(msg.get_memory_size()); - } - - if tx.send((msg, reservation)).is_err() { - return; - }; - } - })) - } - - UnboundedReceiverStream::new(rx).map(move |(msg, _reservation)| { - // keep the tasks alive as long as the stream lives - let _ = &tasks; - msg - }) -} - -pub(super) trait MemoryFootPrint { - fn get_memory_size(&self) -> usize; -} - -impl MemoryFootPrint for RecordBatch { - fn get_memory_size(&self) -> usize { - self.get_array_memory_size() - } -} - -#[cfg(test)] -mod tests { - use crate::execution_plans::common::{MemoryFootPrint, spawn_select_all}; - use datafusion::execution::memory_pool::{MemoryPool, UnboundedMemoryPool}; - use std::error::Error; - use std::sync::Arc; - use tokio_stream::StreamExt; - - #[tokio::test] - async fn memory_reservation() -> Result<(), Box> { - let pool: Arc = Arc::new(UnboundedMemoryPool::default()); - - let mut stream = spawn_select_all( - vec![ - futures::stream::iter(vec![Ok::<_, String>(1), Ok(2), Ok(3)]), - futures::stream::iter(vec![Ok(4), Ok(5)]), - ], - Arc::clone(&pool), - ); - tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; - let reserved = pool.reserved(); - assert_eq!(reserved, 15); - - for i in [1, 2, 3] { - let n = stream.next().await.unwrap()?; - assert_eq!(i, n) - } - - let reserved = pool.reserved(); - assert_eq!(reserved, 9); - - drop(stream); - - let reserved = pool.reserved(); - assert_eq!(reserved, 0); - - Ok(()) - } - - impl MemoryFootPrint for usize { - fn get_memory_size(&self) -> usize { - *self - } - } -} diff --git a/src/execution_plans/network_coalesce.rs b/src/execution_plans/network_coalesce.rs index bafc09d7..3ef3b4e4 100644 --- a/src/execution_plans/network_coalesce.rs +++ b/src/execution_plans/network_coalesce.rs @@ -1,34 +1,20 @@ use crate::common::require_one_child; -use crate::config_extension_ext::ContextGrpcMetadata; use crate::distributed_planner::NetworkBoundary; -use crate::execution_plans::common::{ - manually_propagate_distributed_config, scale_partitioning_props, spawn_select_all, -}; -use crate::flight_service::DoGet; -use crate::metrics::MetricsCollectingStream; +use crate::execution_plans::common::scale_partitioning_props; +use crate::flight_service::WorkerConnectionPool; use crate::metrics::proto::MetricsSetProto; -use crate::networking::get_distributed_channel_resolver; -use crate::protobuf::{StageKey, map_flight_to_datafusion_error, map_status_to_datafusion_error}; +use crate::protobuf::{AppMetadata, StageKey}; use crate::stage::{MaybeEncodedPlan, Stage}; -use crate::{DistributedConfig, DistributedTaskContext, ExecutionTask}; -use arrow_flight::Ticket; -use arrow_flight::decode::FlightRecordBatchStream; -use arrow_flight::error::FlightError; -use bytes::Bytes; +use crate::{DistributedTaskContext, ExecutionTask}; use dashmap::DashMap; -use datafusion::common::{exec_err, internal_err, plan_err}; -use datafusion::error::DataFusionError; +use datafusion::common::{exec_err, plan_err}; +use datafusion::error::Result; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; -use futures::{StreamExt, TryFutureExt, TryStreamExt}; -use http::Extensions; -use prost::Message; use std::any::Any; -use std::fmt::Formatter; +use std::fmt::{Debug, Formatter}; use std::sync::Arc; -use tonic::Request; -use tonic::metadata::MetadataMap; use uuid::Uuid; /// [ExecutionPlan] that coalesces partitions from multiple tasks into a single task without @@ -70,6 +56,7 @@ pub struct NetworkCoalesceExec { /// the properties we advertise for this execution plan pub(crate) properties: PlanProperties, pub(crate) input_stage: Stage, + pub(crate) worker_connections: WorkerConnectionPool, /// metrics_collection is used to collect metrics from child tasks. It is initially /// instantiated as an empty [DashMap] (see `try_decode` in `distributed_codec.rs`). /// Metrics are populated here via [NetworkCoalesceExec::execute]. @@ -94,7 +81,7 @@ impl NetworkCoalesceExec { num: usize, task_count: usize, input_task_count: usize, - ) -> Result { + ) -> Result { if task_count > 1 { return plan_err!( "NetworkCoalesceExec cannot be executed in more than one task, {task_count} were passed." @@ -108,6 +95,7 @@ impl NetworkCoalesceExec { plan: MaybeEncodedPlan::Decoded(input), tasks: vec![ExecutionTask { url: None }; input_task_count], }, + worker_connections: WorkerConnectionPool::new(input_task_count), metrics_collection: Default::default(), }) } @@ -118,10 +106,7 @@ impl NetworkBoundary for NetworkCoalesceExec { &self.input_stage } - fn with_input_stage( - &self, - input_stage: Stage, - ) -> datafusion::common::Result> { + fn with_input_stage(&self, input_stage: Stage) -> Result> { let mut self_clone = self.clone(); self_clone.input_stage = input_stage; Ok(Arc::new(self_clone)) @@ -163,7 +148,7 @@ impl ExecutionPlan for NetworkCoalesceExec { fn with_new_children( self: Arc, children: Vec>, - ) -> Result, DataFusionError> { + ) -> Result> { let mut self_clone = self.as_ref().clone(); self_clone.input_stage.plan = MaybeEncodedPlan::Decoded(require_one_child(children)?); Ok(Arc::new(self_clone)) @@ -173,83 +158,40 @@ impl ExecutionPlan for NetworkCoalesceExec { &self, partition: usize, context: Arc, - ) -> Result { - // get the channel manager and current stage from our context - let channel_resolver = get_distributed_channel_resolver(context.as_ref()); - - let d_cfg = DistributedConfig::from_config_options(context.session_config().options())?; - let retrieve_metrics = d_cfg.collect_metrics; - - let input_stage = &self.input_stage; - let encoded_input_plan = input_stage.plan.encoded()?; - - let context_headers = ContextGrpcMetadata::headers_from_ctx(&context); + ) -> Result { let task_context = DistributedTaskContext::from_ctx(&context); if task_context.task_index > 0 { return exec_err!("NetworkCoalesceExec cannot be executed in more than one task"); } let partitions_per_task = - self.properties().partitioning.partition_count() / input_stage.tasks.len(); + self.properties().partitioning.partition_count() / self.input_stage.tasks.len(); let target_task = partition / partitions_per_task; let target_partition = partition % partitions_per_task; - // TODO: this propagation should be automatic https://github.com/datafusion-contrib/datafusion-distributed/issues/247 - let context_headers = manually_propagate_distributed_config(context_headers, d_cfg); - let ticket = Request::from_parts( - MetadataMap::from_headers(context_headers), - Extensions::default(), - Ticket { - ticket: DoGet { - plan_proto: encoded_input_plan.clone(), - target_partition: target_partition as u64, - stage_key: Some(StageKey::new( - Bytes::from(input_stage.query_id.as_bytes().to_vec()), - input_stage.num as u64, - target_task as u64, - )), - target_task_index: target_task as u64, - target_task_count: input_stage.tasks.len() as u64, + let worker_connection = self.worker_connections.get_or_init_worker_connection( + &self.input_stage, + 0..partitions_per_task, + target_task, + &context, + )?; + + let metrics_collection = Arc::clone(&self.metrics_collection); + + let stream = worker_connection.stream_partition(target_partition, move |meta| { + if let Some(AppMetadata::MetricsCollection(m)) = meta.content { + for task_metrics in m.tasks { + if let Some(stage_key) = task_metrics.stage_key { + metrics_collection.insert(stage_key, task_metrics.metrics); + }; } - .encode_to_vec() - .into(), - }, - ); - - let Some(task) = input_stage.tasks.get(target_task) else { - return internal_err!("ProgrammingError: Task {target_task} not found"); - }; - - let Some(url) = task.url.clone() else { - return internal_err!("NetworkCoalesceExec: task is unassigned, cannot proceed"); - }; - - let metrics_collection_capture = self.metrics_collection.clone(); - let stream = async move { - let mut client = channel_resolver.get_flight_client_for_url(&url).await?; - let stream = client - .do_get(ticket) - .await - .map_err(map_status_to_datafusion_error)? - .into_inner() - .map_err(|err| FlightError::Tonic(Box::new(err))); - - let stream = if retrieve_metrics { - MetricsCollectingStream::new(stream, metrics_collection_capture).left_stream() - } else { - stream.right_stream() - }; - - Ok(FlightRecordBatchStream::new_from_flight_data(stream) - .map_err(map_flight_to_datafusion_error)) - } - .try_flatten_stream() - .boxed(); + } + })?; Ok(Box::pin(RecordBatchStreamAdapter::new( self.schema(), - spawn_select_all(vec![stream], Arc::clone(context.memory_pool())), + stream, ))) } } diff --git a/src/execution_plans/network_shuffle.rs b/src/execution_plans/network_shuffle.rs index dcef4f47..89d2a40b 100644 --- a/src/execution_plans/network_shuffle.rs +++ b/src/execution_plans/network_shuffle.rs @@ -1,23 +1,13 @@ use crate::common::require_one_child; -use crate::config_extension_ext::ContextGrpcMetadata; -use crate::execution_plans::common::{ - manually_propagate_distributed_config, scale_partitioning, spawn_select_all, -}; -use crate::flight_service::DoGet; -use crate::metrics::MetricsCollectingStream; +use crate::execution_plans::common::scale_partitioning; +use crate::flight_service::WorkerConnectionPool; use crate::metrics::proto::MetricsSetProto; -use crate::networking::get_distributed_channel_resolver; -use crate::protobuf::StageKey; -use crate::protobuf::{map_flight_to_datafusion_error, map_status_to_datafusion_error}; +use crate::protobuf::{AppMetadata, StageKey}; use crate::stage::{MaybeEncodedPlan, Stage}; -use crate::{DistributedConfig, DistributedTaskContext, ExecutionTask, NetworkBoundary}; -use arrow_flight::Ticket; -use arrow_flight::decode::FlightRecordBatchStream; -use arrow_flight::error::FlightError; -use bytes::Bytes; +use crate::{DistributedTaskContext, ExecutionTask, NetworkBoundary}; use dashmap::DashMap; use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; -use datafusion::common::{internal_datafusion_err, plan_err}; +use datafusion::common::{Result, plan_err}; use datafusion::error::DataFusionError; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_expr::Partitioning; @@ -26,14 +16,9 @@ use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, }; -use futures::{StreamExt, TryFutureExt, TryStreamExt}; -use http::Extensions; -use prost::Message; use std::any::Any; use std::fmt::Formatter; use std::sync::Arc; -use tonic::Request; -use tonic::metadata::MetadataMap; use uuid::Uuid; /// [ExecutionPlan] implementation that shuffles data across the network in a distributed context. @@ -121,6 +106,7 @@ pub struct NetworkShuffleExec { /// the properties we advertise for this execution plan pub(crate) properties: PlanProperties, pub(crate) input_stage: Stage, + pub(crate) worker_connections: WorkerConnectionPool, /// metrics_collection is used to collect metrics from child tasks. It is initially /// instantiated as an empty [DashMap] (see `try_decode` in `distributed_codec.rs`). /// Metrics are populated here via [NetworkCoalesceExec::execute]. @@ -176,6 +162,7 @@ impl NetworkShuffleExec { plan: MaybeEncodedPlan::Decoded(transformed.data), tasks: vec![ExecutionTask { url: None }; input_task_count], }, + worker_connections: WorkerConnectionPool::new(input_task_count), properties: input.properties().clone(), metrics_collection: Default::default(), }) @@ -187,10 +174,7 @@ impl NetworkBoundary for NetworkShuffleExec { &self.input_stage } - fn with_input_stage( - &self, - input_stage: Stage, - ) -> datafusion::common::Result> { + fn with_input_stage(&self, input_stage: Stage) -> Result> { let mut self_clone = self.clone(); self_clone.input_stage = input_stage; Ok(Arc::new(self_clone)) @@ -243,75 +227,34 @@ impl ExecutionPlan for NetworkShuffleExec { partition: usize, context: Arc, ) -> Result { - // get the channel manager and current stage from our context - let channel_resolver = get_distributed_channel_resolver(context.as_ref()); - - let d_cfg = DistributedConfig::from_config_options(context.session_config().options())?; - let retrieve_metrics = d_cfg.collect_metrics; - - let input_stage = &self.input_stage; - let encoded_input_plan = input_stage.plan.encoded()?; - - let input_stage_tasks = input_stage.tasks.to_vec(); - let input_task_count = input_stage_tasks.len(); - let input_stage_num = input_stage.num as u64; - let query_id = Bytes::from(input_stage.query_id.as_bytes().to_vec()); - - let context_headers = ContextGrpcMetadata::headers_from_ctx(&context); let task_context = DistributedTaskContext::from_ctx(&context); let off = self.properties.partitioning.partition_count() * task_context.task_index; - // TODO: this propagation should be automatic https://github.com/datafusion-contrib/datafusion-distributed/issues/247 - let context_headers = manually_propagate_distributed_config(context_headers, d_cfg); - let stream = input_stage_tasks.into_iter().enumerate().map(|(i, task)| { - let channel_resolver = channel_resolver.clone(); + let mut streams = Vec::with_capacity(self.input_stage.tasks.len()); + for input_task_index in 0..self.input_stage.tasks.len() { + let worker_connection = self.worker_connections.get_or_init_worker_connection( + &self.input_stage, + off..(off + self.properties.partitioning.partition_count()), + input_task_index, + &context, + )?; - let ticket = Request::from_parts( - MetadataMap::from_headers(context_headers.clone()), - Extensions::default(), - Ticket { - ticket: DoGet { - plan_proto: encoded_input_plan.clone(), - target_partition: (off + partition) as u64, - stage_key: Some(StageKey::new(query_id.clone(), input_stage_num, i as u64)), - target_task_index: i as u64, - target_task_count: input_task_count as u64, + let metrics_collection = Arc::clone(&self.metrics_collection); + let stream = worker_connection.stream_partition(off + partition, move |meta| { + if let Some(AppMetadata::MetricsCollection(m)) = meta.content { + for task_metrics in m.tasks { + if let Some(stage_key) = task_metrics.stage_key { + metrics_collection.insert(stage_key, task_metrics.metrics); + }; } - .encode_to_vec() - .into(), - }, - ); - - let metrics_collection_capture = self.metrics_collection.clone(); - async move { - let url = task.url.ok_or(internal_datafusion_err!( - "NetworkShuffleExec: task is unassigned, cannot proceed" - ))?; - - let mut client = channel_resolver.get_flight_client_for_url(&url).await?; - let stream = client - .do_get(ticket) - .await - .map_err(map_status_to_datafusion_error)? - .into_inner() - .map_err(|err| FlightError::Tonic(Box::new(err))); - - let stream = if retrieve_metrics { - MetricsCollectingStream::new(stream, metrics_collection_capture).left_stream() - } else { - stream.right_stream() - }; - - Ok(FlightRecordBatchStream::new_from_flight_data(stream) - .map_err(map_flight_to_datafusion_error)) - } - .try_flatten_stream() - .boxed() - }); + } + })?; + streams.push(stream); + } Ok(Box::pin(RecordBatchStreamAdapter::new( self.schema(), - spawn_select_all(stream.collect(), Arc::clone(context.memory_pool())), + futures::stream::select_all(streams), ))) } } diff --git a/src/flight_service/do_get.rs b/src/flight_service/do_get.rs index 9921722f..7e7d60d6 100644 --- a/src/flight_service/do_get.rs +++ b/src/flight_service/do_get.rs @@ -11,20 +11,19 @@ use crate::protobuf::{ datafusion_error_to_tonic_status, }; use crate::{DistributedConfig, DistributedTaskContext}; -use arrow_flight::FlightData; use arrow_flight::Ticket; -use arrow_flight::encode::{DictionaryHandling, FlightDataEncoderBuilder}; +use arrow_flight::encode::{DictionaryHandling, FlightDataEncoder, FlightDataEncoderBuilder}; use arrow_flight::error::FlightError; use arrow_flight::flight_service_server::FlightService; use arrow_select::dictionary::garbage_collect_any_dictionary; use bytes::Bytes; use datafusion::arrow::array::{Array, AsArray, RecordBatch}; +use crate::flight_service::spawn_select_all::spawn_select_all; use datafusion::common::exec_datafusion_err; use datafusion::error::DataFusionError; -use datafusion::execution::SessionStateBuilder; +use datafusion::execution::{SendableRecordBatchStream, SessionStateBuilder}; use datafusion::physical_plan::ExecutionPlan; -use datafusion::prelude::SessionContext; use datafusion_proto::physical_plan::AsExecutionPlan; use datafusion_proto::protobuf::PhysicalPlanNode; use futures::TryStreamExt; @@ -43,14 +42,17 @@ pub struct DoGet { pub target_task_index: u64, #[prost(uint64, tag = "3")] pub target_task_count: u64, - /// the partition number we want to execute + /// lower bound for the list of partitions to execute (inclusive). #[prost(uint64, tag = "4")] - pub target_partition: u64, + pub target_partition_start: u64, + /// upper bound for the list of partitions to execute (exclusive). + #[prost(uint64, tag = "5")] + pub target_partition_end: u64, /// The stage key that identifies the stage. This is useful to keep /// outside of the stage proto as it is used to store the stage /// and we may not need to deserialize the entire stage proto /// if we already have stored it - #[prost(message, optional, tag = "5")] + #[prost(message, optional, tag = "6")] pub stage_key: Option, } @@ -90,10 +92,8 @@ impl Worker { .map_err(|err| datafusion_error_to_tonic_status(&err))?; let codec = DistributedCodec::new_combined_with_user(session_state.config()); - let ctx = SessionContext::new_with_state(session_state.clone()); + let task_ctx = session_state.task_ctx(); - // There's only 1 `StageExec` responsible for all requests that share the same `stage_key`, - // so here we either retrieve the existing one or create a new one if it does not exist. let key = doget.stage_key.ok_or_else(missing("stage_key"))?; let once = self .task_data_entries @@ -102,7 +102,7 @@ impl Worker { let stage_data = once .get_or_try_init(|| async { let proto_node = PhysicalPlanNode::try_decode(doget.plan_proto.as_ref())?; - let mut plan = proto_node.try_into_physical_plan(&ctx.task_ctx(), &codec)?; + let mut plan = proto_node.try_into_physical_plan(&task_ctx, &codec)?; for hook in self.hooks.on_plan.iter() { plan = hook(plan) } @@ -118,7 +118,6 @@ impl Worker { .map_err(|err| Status::invalid_argument(format!("Cannot decode stage proto: {err}")))?; let plan = Arc::clone(&stage_data.plan); - // Find out which partition group we are executing let cfg = session_state.config_mut(); let d_cfg = set_distributed_option_extension_from_headers::(cfg, &headers) @@ -131,64 +130,62 @@ impl Worker { })); let partition_count = plan.properties().partitioning.partition_count(); - let target_partition = doget.target_partition as usize; let plan_name = plan.name(); - if target_partition >= partition_count { - return Err(datafusion_error_to_tonic_status(&exec_datafusion_err!( - "partition {target_partition} not available. The head plan {plan_name} of the stage just has {partition_count} partitions" - ))); - } - // Rather than executing the `StageExec` itself, we want to execute the inner plan instead, - // as executing `StageExec` performs some worker assignation that should have already been - // done in the head stage. - let stream = plan - .execute(doget.target_partition as usize, session_state.task_ctx()) - .map_err(|err| Status::internal(format!("Error executing stage plan: {err:#?}")))?; - - let schema = stream.schema().clone(); - - // Apply garbage collection of dictionary and view arrays before sending over the network - let stream = stream.and_then(|rb| std::future::ready(garbage_collect_arrays(rb))); - - let stream = FlightDataEncoderBuilder::new() - .with_schema(schema) - // This tells the encoder to send dictionaries across the wire as-is. - // The alternative (`DictionaryHandling::Hydrate`) would expand the dictionaries - // into their value types, which can potentially blow up the size of the data transfer. - // The main reason to use `DictionaryHandling::Hydrate` is for compatibility with clients - // that do not support dictionaries, but since we are using the same server/client on both - // sides, we can safely use `DictionaryHandling::Resend`. - // Note that we do garbage collection of unused dictionary values above, so we are not sending - // unused dictionary values over the wire. - .with_dictionary_handling(DictionaryHandling::Resend) - // Set max flight data size to unlimited. - // This requires servers and clients to also be configured to handle unlimited sizes. - // Using unlimited sizes avoids splitting RecordBatches into multiple FlightData messages, - // which could add significant overhead for large RecordBatches. - // The only reason to split them really is if the client/server are configured with a message size limit, - // which mainly makes sense in a public network scenario where you want to avoid DoS attacks. - // Since all of our Arrow Flight communication happens within trusted data plane networks, - // we can safely use unlimited sizes here. - .with_max_flight_data_size(usize::MAX) - .build(stream.map_err(|err| { - FlightError::Tonic(Box::new(datafusion_error_to_tonic_status(&err))) - })); - - let task_data_entries = Arc::clone(&self.task_data_entries); - let num_partitions_remaining = Arc::clone(&stage_data.num_partitions_remaining); - - let stream = map_last_stream(stream, move |last| { - if num_partitions_remaining.fetch_sub(1, Ordering::SeqCst) == 1 { - task_data_entries.remove(key.clone()); - return if send_metrics { - last.and_then(|el| collect_and_create_metrics_flight_data(key, plan, el)) - } else { - last - }; + // Execute all the requested partitions at once, and collect all the streams so that they + // can be merged into a single one at the end of this function. + let n_streams = doget.target_partition_end - doget.target_partition_start; + let mut streams = Vec::with_capacity(n_streams as usize); + for partition in doget.target_partition_start..doget.target_partition_end { + if partition >= partition_count as u64 { + return Err(datafusion_error_to_tonic_status(&exec_datafusion_err!( + "partition {partition} not available. The head plan {plan_name} of the stage just has {partition_count} partitions" + ))); } - last - }); + + let stream = plan + .execute(partition as usize, session_state.task_ctx()) + .map_err(|err| Status::internal(format!("Error executing stage plan: {err:#?}")))?; + + let stream = build_flight_data_stream(stream); + + let task_data_entries = Arc::clone(&self.task_data_entries); + let num_partitions_remaining = Arc::clone(&stage_data.num_partitions_remaining); + + let key = key.clone(); + let plan = Arc::clone(&plan); + let stream = map_last_stream(stream, move |msg, last_msg_in_stream| { + // For each FlightData produced by this stream, mark it with the appropriate + // partition. This stream will be merged with several others from other partitions, + // so marking it with the original partition allows it to be deconstructed into + // the original per-partition streams in later steps. + let mut flight_data = FlightAppMetadata::new(partition); + + if last_msg_in_stream { + // If it's the last message from the last partition, clean up the entry from + // the TTLMap and send the collected metrics. + if num_partitions_remaining.fetch_sub(1, Ordering::SeqCst) == 1 { + task_data_entries.remove(key.clone()); + if send_metrics { + // Last message of the last partition. This is the moment to send + // the metrics back. + flight_data.set_content(collect_and_create_metrics_flight_data( + key.clone(), + plan.clone(), + )?); + } + } + } + + msg.map(|v| v.with_app_metadata(flight_data.encode_to_vec())) + }); + streams.push(stream) + } + + // Merge all the per-partition streams into one. Each message in the stream is marked with + // the original partition, so they can be reconstructed at the other side of the boundary. + let memory_pool = Arc::clone(&session_state.runtime_env().memory_pool); + let stream = spawn_select_all(streams, memory_pool); Ok(Response::new(Box::pin(stream.map_err(|err| match err { FlightError::Tonic(status) => *status, @@ -197,6 +194,37 @@ impl Worker { } } +fn build_flight_data_stream(stream: SendableRecordBatchStream) -> FlightDataEncoder { + FlightDataEncoderBuilder::new() + .with_schema(stream.schema()) + // This tells the encoder to send dictionaries across the wire as-is. + // The alternative (`DictionaryHandling::Hydrate`) would expand the dictionaries + // into their value types, which can potentially blow up the size of the data transfer. + // The main reason to use `DictionaryHandling::Hydrate` is for compatibility with clients + // that do not support dictionaries, but since we are using the same server/client on both + // sides, we can safely use `DictionaryHandling::Resend`. + // Note that we do garbage collection of unused dictionary values above, so we are not sending + // unused dictionary values over the wire. + .with_dictionary_handling(DictionaryHandling::Resend) + // Set max flight data size to unlimited. + // This requires servers and clients to also be configured to handle unlimited sizes. + // Using unlimited sizes avoids splitting RecordBatches into multiple FlightData messages, + // which could add significant overhead for large RecordBatches. + // The only reason to split them really is if the client/server are configured with a message size limit, + // which mainly makes sense in a public network scenario where you want to avoid DoS attacks. + // Since all of our Arrow Flight communication happens within trusted data plane networks, + // we can safely use unlimited sizes here. + .with_max_flight_data_size(usize::MAX) + .build( + stream + // Apply garbage collection of dictionary and view arrays before sending over the network + .and_then(|rb| std::future::ready(garbage_collect_arrays(rb))) + .map_err(|err| { + FlightError::Tonic(Box::new(datafusion_error_to_tonic_status(&err))) + }), + ) +} + fn missing(field: &'static str) -> impl FnOnce() -> Status { move || Status::invalid_argument(format!("Missing field '{field}'")) } @@ -205,8 +233,7 @@ fn missing(field: &'static str) -> impl FnOnce() -> Status { fn collect_and_create_metrics_flight_data( stage_key: StageKey, plan: Arc, - incoming: FlightData, -) -> Result { +) -> Result { // Get the metrics for the task executed on this worker + child tasks. let mut result = TaskMetricsCollector::new() .collect(plan) @@ -235,18 +262,9 @@ fn collect_and_create_metrics_flight_data( }); } - let flight_app_metadata = FlightAppMetadata { - content: Some(AppMetadata::MetricsCollection(MetricsCollection { - tasks: task_metrics_set, - })), - }; - - let mut buf = vec![]; - flight_app_metadata - .encode(&mut buf) - .map_err(|err| FlightError::ProtocolError(err.to_string()))?; - - Ok(incoming.with_app_metadata(buf)) + Ok(AppMetadata::MetricsCollection(MetricsCollection { + tasks: task_metrics_set, + })) } /// Garbage collects values sub-arrays. @@ -335,7 +353,8 @@ mod tests { plan_proto, target_task_index: task_number, target_task_count: num_tasks, - target_partition: partition, + target_partition_start: partition, + target_partition_end: partition + 1, stage_key: Some(stage_key), }; diff --git a/src/flight_service/mod.rs b/src/flight_service/mod.rs index 5c31b1b3..aca36c66 100644 --- a/src/flight_service/mod.rs +++ b/src/flight_service/mod.rs @@ -1,7 +1,10 @@ mod do_get; mod session_builder; +mod spawn_select_all; mod worker; -pub(crate) use do_get::DoGet; +mod worker_connection_pool; + +pub(crate) use worker_connection_pool::WorkerConnectionPool; pub use session_builder::{ DefaultSessionBuilder, MappedWorkerSessionBuilder, MappedWorkerSessionBuilderExt, diff --git a/src/flight_service/spawn_select_all.rs b/src/flight_service/spawn_select_all.rs new file mode 100644 index 00000000..61f205b4 --- /dev/null +++ b/src/flight_service/spawn_select_all.rs @@ -0,0 +1,109 @@ +use arrow_flight::FlightData; +use datafusion::arrow::array::RecordBatch; +use datafusion::common::runtime::SpawnedTask; +use datafusion::execution::memory_pool::{MemoryConsumer, MemoryPool}; +use futures::{Stream, StreamExt}; +use std::sync::Arc; +use tokio_stream::wrappers::UnboundedReceiverStream; + +/// Consumes all the provided streams in parallel sending their produced messages to a single +/// queue in random order. The resulting queue is returned as a stream. +pub(crate) fn spawn_select_all( + inner: Vec, + pool: Arc, +) -> impl Stream> +where + T: Stream> + Send + Unpin + 'static, + El: MemoryFootPrint + Send + 'static, + Err: Send + 'static, +{ + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + + let mut tasks = vec![]; + for mut t in inner { + let tx = tx.clone(); + let pool = Arc::clone(&pool); + let consumer = MemoryConsumer::new("NetworkBoundary"); + + tasks.push(SpawnedTask::spawn(async move { + while let Some(msg) = t.next().await { + let mut reservation = consumer.clone_with_new_id().register(&pool); + if let Ok(msg) = &msg { + reservation.grow(msg.get_memory_size()); + } + + if tx.send((msg, reservation)).is_err() { + return; + }; + } + })) + } + + UnboundedReceiverStream::new(rx).map(move |(msg, _reservation)| { + // keep the tasks alive as long as the stream lives + let _ = &tasks; + msg + }) +} + +pub(crate) trait MemoryFootPrint { + fn get_memory_size(&self) -> usize; +} + +impl MemoryFootPrint for RecordBatch { + fn get_memory_size(&self) -> usize { + self.get_array_memory_size() + } +} + +impl MemoryFootPrint for FlightData { + fn get_memory_size(&self) -> usize { + self.data_header.len() + self.data_body.len() + self.app_metadata.len() + } +} + +#[cfg(test)] +mod tests { + use super::{MemoryFootPrint, spawn_select_all}; + use datafusion::execution::memory_pool::{MemoryPool, UnboundedMemoryPool}; + use std::error::Error; + use std::sync::Arc; + use tokio_stream::StreamExt; + + #[tokio::test] + async fn memory_reservation() -> Result<(), Box> { + let pool: Arc = Arc::new(UnboundedMemoryPool::default()); + + let mut stream = spawn_select_all( + vec![ + futures::stream::iter(vec![Ok::<_, String>(1), Ok(2), Ok(3)]), + futures::stream::iter(vec![Ok(4), Ok(5)]), + ], + Arc::clone(&pool), + ); + tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; + let reserved = pool.reserved(); + assert_eq!(reserved, 15); + + for i in [1, 2, 3] { + let n = stream.next().await.unwrap()?; + assert_eq!(i, n) + } + + let reserved = pool.reserved(); + assert_eq!(reserved, 9); + + drop(stream); + + let reserved = pool.reserved(); + assert_eq!(reserved, 0); + + Ok(()) + } + + impl MemoryFootPrint for usize { + fn get_memory_size(&self) -> usize { + *self + } + } +} diff --git a/src/flight_service/worker_connection_pool.rs b/src/flight_service/worker_connection_pool.rs new file mode 100644 index 00000000..c86e0ebf --- /dev/null +++ b/src/flight_service/worker_connection_pool.rs @@ -0,0 +1,282 @@ +use crate::config_extension_ext::ContextGrpcMetadata; +use crate::flight_service::do_get::DoGet; +use crate::networking::get_distributed_channel_resolver; +use crate::protobuf::{ + FlightAppMetadata, StageKey, datafusion_error_to_tonic_status, map_flight_to_datafusion_error, +}; +use crate::{DistributedConfig, Stage}; +use arrow_flight::decode::FlightRecordBatchStream; +use arrow_flight::error::FlightError; +use arrow_flight::{FlightData, Ticket}; +use bytes::Bytes; +use dashmap::DashMap; +use datafusion::arrow::array::RecordBatch; +use datafusion::common::runtime::SpawnedTask; +use datafusion::common::{DataFusionError, Result, internal_err}; +use datafusion::execution::TaskContext; +use datafusion::execution::memory_pool::{MemoryConsumer, MemoryReservation}; +use futures::{Stream, TryStreamExt}; +use http::{Extensions, HeaderMap}; +use prost::Message; +use std::fmt::{Debug, Formatter}; +use std::ops::Range; +use std::sync::{Arc, OnceLock}; +use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; +use tokio_stream::StreamExt; +use tokio_stream::wrappers::UnboundedReceiverStream; +use tonic::metadata::MetadataMap; +use tonic::{Request, Status}; + +/// Holds a list of lazily initialized [WorkerConnection]s. Each position in the underlying +/// `connections` vector corresponds to the connection to one worker. It assumes a 1:1 mapping +/// between worker and tasks, and upon calling [WorkerConnectionPool::get_or_init_worker_connection] +/// it will initialize the corresponding position in the vector matching the provided `target_task` +/// index. +pub(crate) struct WorkerConnectionPool { + connections: Vec>>>, +} + +impl WorkerConnectionPool { + /// Builds a new [WorkerConnectionPool] with as many empty slots for [WorkerConnection]s as + /// the provided `input_tasks`. + pub(crate) fn new(input_tasks: usize) -> Self { + let mut connections = Vec::with_capacity(input_tasks); + for _ in 0..input_tasks { + connections.push(OnceLock::new()); + } + Self { connections } + } + + /// Lazily initializes the [WorkerConnection] corresponding to the provided `target_task` + /// (therefore maintaining one independent [WorkerConnection] per `target_task`), and + /// returns it. + pub(crate) fn get_or_init_worker_connection( + &self, + input_stage: &Stage, + target_partitions: Range, + target_task: usize, + ctx: &Arc, + ) -> Result<&WorkerConnection> { + let Some(worker_connection) = self.connections.get(target_task) else { + return internal_err!( + "WorkerConnections: Task index {target_task} not found, only have {} tasks", + self.connections.len() + ); + }; + + let conn = worker_connection.get_or_init(|| { + WorkerConnection::init(input_stage, target_partitions, target_task, ctx) + .map_err(Arc::new) + }); + + match conn { + Ok(v) => Ok(v), + Err(err) => Err(DataFusionError::Shared(Arc::clone(err))), + } + } +} + +type WorkerMsg = Result<(FlightData, FlightAppMetadata, MemoryReservation), Status>; + +/// Represents a connection to one [Worker]. Network boundaries will use this for streaming +/// data from single partitions while the actual network communication is handling all the partitions +/// under the hood. +/// +/// This is done so that, rather than issuing one gRPC stream per partition, we issue one gRPC stream +/// per group of partitions, and we multiplex streamed record batches locally to in-memory channels. +/// +/// Even if Tonic can perfectly multiplex and interleave messages from different gRPC streams through +/// the same underlying TCP connection, there do is some overhead in having one gRPC stream per +/// partition VS a single gRPC stream interleaving multiple partitions. The whole serialized plan +/// needs to be sent over the wire on every gRPC call, so the less gRPC calls we do the better. +pub(crate) struct WorkerConnection { + task: Arc>, + per_partition_rx: DashMap>, +} + +impl WorkerConnection { + fn init( + input_stage: &Stage, + target_partition_range: Range, + target_task: usize, + ctx: &Arc, + ) -> Result { + let channel_resolver = get_distributed_channel_resolver(ctx.as_ref()); + let d_cfg = DistributedConfig::from_config_options(ctx.session_config().options())?; + + let context_headers = ContextGrpcMetadata::headers_from_ctx(ctx); + // TODO: this propagation should be automatic https://github.com/datafusion-contrib/datafusion-distributed/issues/247 + let context_headers = manually_propagate_distributed_config(context_headers, d_cfg); + let ticket = Request::from_parts( + MetadataMap::from_headers(context_headers), + Extensions::default(), + Ticket { + ticket: DoGet { + plan_proto: Bytes::clone(input_stage.plan.encoded()?), + target_partition_start: target_partition_range.start as u64, + target_partition_end: target_partition_range.end as u64, + stage_key: Some(StageKey::new( + Bytes::from(input_stage.query_id.as_bytes().to_vec()), + input_stage.num as u64, + target_task as u64, + )), + target_task_index: target_task as u64, + target_task_count: input_stage.tasks.len() as u64, + } + .encode_to_vec() + .into(), + }, + ); + + let Some(task) = input_stage.tasks.get(target_task) else { + return internal_err!("ProgrammingError: Task {target_task} not found"); + }; + let Some(url) = task.url.clone() else { + return internal_err!("ProgrammingError: task is unassigned, cannot proceed"); + }; + + // The senders and receivers are unbounded queues used for multiplexing the record + // batches sent through the single gRPC stream into one stream per partition. + // The received record batches contain information of the partition to which they belong, + // so we use that for determining where to put them. + let mut per_partition_tx = Vec::with_capacity(target_partition_range.len()); + let per_partition_rx = DashMap::with_capacity(target_partition_range.len()); + for partition in target_partition_range.clone() { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel::(); + per_partition_tx.push(tx); + per_partition_rx.insert(partition, rx); + } + + // We are retaining record batches in memory until they are consumed, so we need to account + // for them in the memory pool. + let memory_pool = Arc::clone(ctx.memory_pool()); + + // This task will pull data from all the partitions in `target_partition_range`, and will + // fan them out to the appropriate `per_partition_rx` based on the "partition" declared + // in each individual record batch flight metadata. + let task = SpawnedTask::spawn(async move { + let mut client = match channel_resolver.get_flight_client_for_url(&url).await { + Ok(v) => v, + Err(err) => { + return fanout(&per_partition_tx, datafusion_error_to_tonic_status(&err)); + } + }; + let mut interleaved_stream = match client.do_get(ticket).await { + Ok(v) => v.into_inner(), + Err(err) => return fanout(&per_partition_tx, err), + }; + + let consumer = MemoryConsumer::new("WorkerConnection"); + + while let Some(msg) = interleaved_stream.next().await { + let msg = match msg { + Ok(v) => v, + Err(err) => return fanout(&per_partition_tx, err), + }; + let flight_metadata = match FlightAppMetadata::decode(msg.app_metadata.as_ref()) { + Ok(v) => v, + Err(err) => { + return fanout(&per_partition_tx, Status::internal(err.to_string())); + } + }; + + let partition = flight_metadata.partition as usize; + // the `per_partition_tx` variable is using a normal `Vec` for storing the + // channel transmitters, so we need to subtract the `target_partition_range.start` + // to the `partition` in order to offset it to the appropriate index. + let sender_i = partition - target_partition_range.start; + + let Some(o_tx) = per_partition_tx.get(sender_i) else { + let msg = format!( + "Received partition {partition} in Flight metadata, but available partitions are {target_partition_range:?}" + ); + return fanout(&per_partition_tx, Status::internal(msg)); + }; + + // We need to send the memory reservation in the same tuple as the actual message + // so that it gets dropped as soon as the message leaves the queue. Dropping the + // memory reservation means releasing the memory from the pool for that specific + // message + let reservation = consumer.clone_with_new_id().register(&memory_pool); + if o_tx.send(Ok((msg, flight_metadata, reservation))).is_err() { + return; // channel closed + }; + } + }); + + Ok(Self { + task: Arc::new(task), + per_partition_rx, + }) + } + + /// Streams the provided `partition` from the remote worker. + /// + /// Note that this does not issue a network request, the actual network request happened before + /// in the init step, and is in charge of handling not only this `partition`, but also all the + /// partitions passed in `target_partition_range`. This method just streams all the record + /// batches belonging to the provided `partition` from an in-memory queue, but what populates + /// this queue is [WorkerConnection::init]. + pub(crate) fn stream_partition( + &self, + partition: usize, + on_metadata: impl Fn(FlightAppMetadata) + Send + Sync + 'static, + ) -> Result> + 'static> { + let Some((_, partition_receiver)) = self.per_partition_rx.remove(&partition) else { + return internal_err!( + "WorkerConnection has no stream for target partition {partition}. Was it already consumed?" + ); + }; + let task = Arc::clone(&self.task); + let stream = UnboundedReceiverStream::new(partition_receiver); + let stream = stream.map_err(|err| FlightError::Tonic(Box::new(err))); + let stream = stream.map_ok(move |(data, meta, reservation)| { + drop(reservation); // <- drop the reservation, freeing memory on the memory pool. + let _ = &task; // <- keep the task that pools data from the network alive. + on_metadata(meta); + data + }); + let stream = FlightRecordBatchStream::new_from_flight_data(stream); + let stream = stream.map_err(map_flight_to_datafusion_error); + Ok(stream) + } +} + +fn fanout(o_txs: &[UnboundedSender], err: Status) { + for o_tx in o_txs { + let _ = o_tx.send(Err(err.clone())); + } +} + +/// Manual propagation of the [DistributedConfig] fields relevant for execution. Can be removed +/// after https://github.com/datafusion-contrib/datafusion-distributed/issues/247 is fixed, as this will become automatic. +pub(super) fn manually_propagate_distributed_config( + mut headers: HeaderMap, + d_cfg: &DistributedConfig, +) -> HeaderMap { + headers.insert( + "distributed.collect_metrics", + d_cfg.collect_metrics.to_string().parse().unwrap(), + ); + headers +} + +impl Debug for WorkerConnectionPool { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WorkerConnections") + .field("num_connections", &self.connections.len()) + .finish() + } +} + +impl Clone for WorkerConnectionPool { + fn clone(&self) -> Self { + Self::new(self.connections.len()) + } +} + +impl Debug for WorkerConnection { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WorkerConnection").finish() + } +} diff --git a/src/metrics/metrics_collecting_stream.rs b/src/metrics/metrics_collecting_stream.rs deleted file mode 100644 index 593e09b2..00000000 --- a/src/metrics/metrics_collecting_stream.rs +++ /dev/null @@ -1,257 +0,0 @@ -use std::pin::Pin; -use std::task::{Context, Poll}; - -use crate::metrics::proto::MetricsSetProto; -use crate::protobuf::StageKey; -use crate::protobuf::{AppMetadata, FlightAppMetadata}; -use arrow_flight::{FlightData, error::FlightError}; -use dashmap::DashMap; -use futures::stream::Stream; -use pin_project::pin_project; -use prost::Message; -use std::sync::Arc; - -/// MetricsCollectingStream wraps a FlightData stream and extracts metrics from app_metadata -/// while passing through all the other FlightData unchanged. -#[pin_project] -pub struct MetricsCollectingStream -where - S: Stream> + Send, -{ - #[pin] - inner: S, - metrics_collection: Arc>>, -} - -impl MetricsCollectingStream -where - S: Stream> + Send, -{ - pub fn new( - stream: S, - metrics_collection: Arc>>, - ) -> Self { - Self { - inner: stream, - metrics_collection, - } - } - - fn extract_metrics_from_flight_data( - metrics_collection: Arc>>, - flight_data: &mut FlightData, - ) -> Result<(), FlightError> { - if flight_data.app_metadata.is_empty() { - return Ok(()); - } - - let metadata = - FlightAppMetadata::decode(flight_data.app_metadata.as_ref()).map_err(|e| { - FlightError::ProtocolError(format!("failed to decode app_metadata: {e}")) - })?; - - let Some(content) = metadata.content else { - return Err(FlightError::ProtocolError( - "expected Some content in app_metadata".to_string(), - )); - }; - - let AppMetadata::MetricsCollection(task_metrics_set) = content; - - for task_metrics in task_metrics_set.tasks { - let Some(stage_key) = task_metrics.stage_key else { - return Err(FlightError::ProtocolError( - "expected Some StageKey in MetricsCollectingStream, got None".to_string(), - )); - }; - metrics_collection.insert(stage_key, task_metrics.metrics); - } - flight_data.app_metadata.clear(); - - Ok(()) - } -} - -impl Stream for MetricsCollectingStream -where - S: Stream> + Send, -{ - type Item = Result; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.project(); - match this.inner.poll_next(cx) { - Poll::Ready(Some(Ok(mut flight_data))) => { - // Extract metrics from app_metadata if present. - match Self::extract_metrics_from_flight_data( - this.metrics_collection.clone(), - &mut flight_data, - ) { - Ok(_) => Poll::Ready(Some(Ok(flight_data))), - Err(e) => Poll::Ready(Some(Err(e))), - } - } - Poll::Ready(Some(Err(err))) => Poll::Ready(Some(Err(err))), - Poll::Ready(None) => Poll::Ready(None), - Poll::Pending => Poll::Pending, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::protobuf::{ - AppMetadata, FlightAppMetadata, MetricsCollection, StageKey, TaskMetrics, - }; - use crate::test_utils::metrics::make_test_metrics_set_proto_from_seed; - use arrow_flight::FlightData; - use futures::stream::{self, StreamExt}; - use prost::{Message, bytes::Bytes}; - - fn assert_protocol_error(result: Result, expected_msg: &str) { - let FlightError::ProtocolError(msg) = result.unwrap_err() else { - panic!("expected FlightError::ProtocolError"); - }; - assert!(msg.contains(expected_msg)); - } - - fn make_flight_data(data: Vec, metadata: Option) -> FlightData { - let metadata_bytes = match metadata { - Some(metadata) => metadata.encode_to_vec().into(), - None => Bytes::new(), - }; - FlightData { - flight_descriptor: None, - data_header: Bytes::new(), - app_metadata: metadata_bytes, - data_body: data.into(), - } - } - - // Tests that metrics are extracted from FlightData. Metrics are always stored per task, so this - // tests creates some random metrics and tasks and puts them in the app_metadata field of - // FlightData message. Then, it streams these messages through the MetricsCollectingStream - // and asserts that the metrics are collected correctly. - #[tokio::test] - async fn test_metrics_collecting_stream_extracts_and_removes_metadata() { - let stage_keys: Vec<_> = (1..3) - .map(|i| StageKey::new(Bytes::from("test_query"), 1, i)) - .collect(); - let app_metadatas = stage_keys - .iter() - .map(|stage_key| FlightAppMetadata { - content: Some(AppMetadata::MetricsCollection(MetricsCollection { - tasks: vec![TaskMetrics { - stage_key: Some(stage_key.clone()), - // use the task number to seed the test metrics set for convenience - metrics: vec![make_test_metrics_set_proto_from_seed( - stage_key.task_number, - 4, - )], - }], - })), - }) - .collect::>(); - - // Create test FlightData messages - some with metadata, some without - let flight_messages = vec![ - make_flight_data(vec![1], Some(app_metadatas[0].clone())), - make_flight_data(vec![2], None), - make_flight_data(vec![3], Some(app_metadatas[1].clone())), - ] - .into_iter() - .map(Ok); - - // Collect all messages from the stream. All should have empty app_metadata. - let metrics_collection = Arc::new(DashMap::new()); - let input_stream = stream::iter(flight_messages); - let collecting_stream = - MetricsCollectingStream::new(input_stream, metrics_collection.clone()); - let collected_messages: Vec = collecting_stream - .map(|result| result.unwrap()) - .collect() - .await; - - // Assert the data is unchanged and app_metadata is cleared - assert_eq!(collected_messages.len(), 3); - assert!( - collected_messages - .iter() - .all(|msg| msg.app_metadata.is_empty()) - ); - - // Verify the data in the messages. - assert_eq!(collected_messages[0].data_body, vec![1]); - assert_eq!(collected_messages[1].data_body, vec![2]); - assert_eq!(collected_messages[2].data_body, vec![3]); - - // Verify the correct metrics were collected - assert_eq!(metrics_collection.len(), 2); - for stage_key in stage_keys { - let collected_metrics = metrics_collection.get(&stage_key).unwrap(); - assert_eq!(collected_metrics.len(), 1); - assert_eq!( - collected_metrics[0], - make_test_metrics_set_proto_from_seed(stage_key.task_number, 4) - ); - } - } - - #[tokio::test] - async fn test_metrics_collecting_stream_error_missing_stage_key() { - let task_metrics_with_no_stage_key = TaskMetrics { - stage_key: None, - metrics: vec![make_test_metrics_set_proto_from_seed(1, 4)], - }; - - let invalid_app_metadata = FlightAppMetadata { - content: Some(AppMetadata::MetricsCollection(MetricsCollection { - tasks: vec![task_metrics_with_no_stage_key], - })), - }; - - let invalid_flight_data = make_flight_data(vec![1], Some(invalid_app_metadata)); - - let error_stream = stream::iter(vec![Ok(invalid_flight_data)]); - let mut collecting_stream = - MetricsCollectingStream::new(error_stream, Arc::new(DashMap::new())); - - let result = collecting_stream.next().await.unwrap(); - assert_protocol_error( - result, - "expected Some StageKey in MetricsCollectingStream, got None", - ); - } - - #[tokio::test] - async fn test_metrics_collecting_stream_error_decoding_metadata() { - let flight_data_with_invalid_metadata = FlightData { - flight_descriptor: None, - data_header: Bytes::new(), - app_metadata: vec![0xFF, 0xFF, 0xFF, 0xFF].into(), // Invalid protobuf data - data_body: vec![4, 5, 6].into(), - }; - - let error_stream = stream::iter(vec![Ok(flight_data_with_invalid_metadata)]); - let mut collecting_stream = - MetricsCollectingStream::new(error_stream, Arc::new(DashMap::new())); - - let result = collecting_stream.next().await.unwrap(); - assert_protocol_error(result, "failed to decode app_metadata"); - } - - #[tokio::test] - async fn test_metrics_collecting_stream_error_propagation() { - let metrics_collection = Arc::new(DashMap::new()); - - // Create a stream that emits an error - should be propagated through - let stream_error = FlightError::ProtocolError("stream error from inner stream".to_string()); - let error_stream = stream::iter(vec![Err(stream_error)]); - let mut collecting_stream = - MetricsCollectingStream::new(error_stream, metrics_collection.clone()); - - let result = collecting_stream.next().await.unwrap(); - assert_protocol_error(result, "stream error from inner stream"); - } -} diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index a9a6e7ff..655add1e 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -1,7 +1,5 @@ -mod metrics_collecting_stream; pub(crate) mod proto; mod task_metrics_collector; mod task_metrics_rewriter; -pub(crate) use metrics_collecting_stream::MetricsCollectingStream; pub(crate) use task_metrics_collector::{MetricsCollectorResult, TaskMetricsCollector}; pub use task_metrics_rewriter::rewrite_distributed_plan_with_metrics; diff --git a/src/protobuf/app_metadata.rs b/src/protobuf/app_metadata.rs index b0de54d1..fa0caa8b 100644 --- a/src/protobuf/app_metadata.rs +++ b/src/protobuf/app_metadata.rs @@ -26,14 +26,29 @@ pub struct TaskMetrics { // FlightAppMetadata represents all types of app_metadata which we use in the distributed execution. #[derive(Clone, PartialEq, ::prost::Message)] pub struct FlightAppMetadata { + #[prost(uint64, tag = "1")] + pub partition: u64, // content should always be Some, but it is optional due to protobuf rules. - #[prost(oneof = "AppMetadata", tags = "1")] + #[prost(oneof = "AppMetadata", tags = "10")] pub content: Option, } +impl FlightAppMetadata { + pub fn new(partition: u64) -> Self { + Self { + partition, + content: None, + } + } + + pub fn set_content(&mut self, content: AppMetadata) { + self.content = Some(content); + } +} + #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum AppMetadata { - #[prost(message, tag = "1")] + #[prost(message, tag = "10")] MetricsCollection(MetricsCollection), // Note: For every additional enum variant, ensure to add tags to [FlightAppMetadata]. ex. `#[prost(oneof = "AppMetadata", tags = "1,2,3")]` etc. // If you don't the proto will compile but you may encounter errors during serialization/deserialization. diff --git a/src/protobuf/distributed_codec.rs b/src/protobuf/distributed_codec.rs index 7d253313..a99e103f 100644 --- a/src/protobuf/distributed_codec.rs +++ b/src/protobuf/distributed_codec.rs @@ -1,5 +1,6 @@ use super::get_distributed_user_codecs; use crate::execution_plans::{ChildrenIsolatorUnionExec, NetworkCoalesceExec}; +use crate::flight_service::WorkerConnectionPool; use crate::stage::{ExecutionTask, MaybeEncodedPlan, Stage}; use crate::{DistributedTaskContext, NetworkBoundary}; use crate::{NetworkShuffleExec, PartitionIsolatorExec}; @@ -402,6 +403,7 @@ fn new_network_hash_shuffle_exec( EmissionType::Incremental, Boundedness::Bounded, ), + worker_connections: WorkerConnectionPool::new(input_stage.tasks.len()), input_stage, metrics_collection: Default::default(), } @@ -432,6 +434,7 @@ fn new_network_coalesce_tasks_exec( EmissionType::Incremental, Boundedness::Bounded, ), + worker_connections: WorkerConnectionPool::new(input_stage.tasks.len()), input_stage, metrics_collection: Default::default(), } diff --git a/src/protobuf/mod.rs b/src/protobuf/mod.rs index 3a61533f..81b444f1 100644 --- a/src/protobuf/mod.rs +++ b/src/protobuf/mod.rs @@ -5,10 +5,7 @@ mod user_codec; pub(crate) use app_metadata::{AppMetadata, FlightAppMetadata, MetricsCollection, TaskMetrics}; pub(crate) use distributed_codec::{DistributedCodec, StageKey}; -pub(crate) use errors::{ - datafusion_error_to_tonic_status, map_flight_to_datafusion_error, - map_status_to_datafusion_error, -}; +pub(crate) use errors::{datafusion_error_to_tonic_status, map_flight_to_datafusion_error}; pub(crate) use user_codec::{ get_distributed_user_codecs, set_distributed_user_codec, set_distributed_user_codec_arc, }; From 7d168416a7d265b945efb29f392ae290c6156273 Mon Sep 17 00:00:00 2001 From: Gabriel Musat Mestre Date: Wed, 14 Jan 2026 08:25:18 +0100 Subject: [PATCH 20/27] Remove ballista from benchmarks --- Cargo.lock | 2025 ++++------------------ benchmarks/Cargo.toml | 16 - benchmarks/cdk/bin/ballista-bench.ts | 97 -- benchmarks/cdk/bin/ballista_executor.rs | 36 - benchmarks/cdk/bin/ballista_http.rs | 124 -- benchmarks/cdk/bin/ballista_scheduler.rs | 62 - benchmarks/cdk/bin/cdk.ts | 2 - benchmarks/cdk/lib/ballista.ts | 212 --- benchmarks/cdk/lib/cdk-stack.ts | 2 - benchmarks/cdk/package.json | 1 - 10 files changed, 367 insertions(+), 2210 deletions(-) delete mode 100644 benchmarks/cdk/bin/ballista-bench.ts delete mode 100644 benchmarks/cdk/bin/ballista_executor.rs delete mode 100644 benchmarks/cdk/bin/ballista_http.rs delete mode 100644 benchmarks/cdk/bin/ballista_scheduler.rs delete mode 100644 benchmarks/cdk/lib/ballista.ts diff --git a/Cargo.lock b/Cargo.lock index 3235fa82..204b8497 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -205,60 +205,25 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" -[[package]] -name = "arrow" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc" -dependencies = [ - "arrow-arith 56.2.0", - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-cast 56.2.0", - "arrow-csv 56.2.0", - "arrow-data 56.2.0", - "arrow-ipc 56.2.0", - "arrow-json 56.2.0", - "arrow-ord 56.2.0", - "arrow-row 56.2.0", - "arrow-schema 56.2.0", - "arrow-select 56.2.0", - "arrow-string 56.2.0", -] - [[package]] name = "arrow" version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb372a7cbcac02a35d3fb7b3fc1f969ec078e871f9bb899bf00a2e1809bec8a3" dependencies = [ - "arrow-arith 57.1.0", - "arrow-array 57.1.0", - "arrow-buffer 57.1.0", - "arrow-cast 57.1.0", - "arrow-csv 57.1.0", - "arrow-data 57.1.0", - "arrow-ipc 57.1.0", - "arrow-json 57.1.0", - "arrow-ord 57.1.0", - "arrow-row 57.1.0", - "arrow-schema 57.1.0", - "arrow-select 57.1.0", - "arrow-string 57.1.0", -] - -[[package]] -name = "arrow-arith" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" -dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", - "chrono", - "num", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", ] [[package]] @@ -267,31 +232,14 @@ version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f377dcd19e440174596d83deb49cd724886d91060c07fec4f67014ef9d54049" dependencies = [ - "arrow-array 57.1.0", - "arrow-buffer 57.1.0", - "arrow-data 57.1.0", - "arrow-schema 57.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "num-traits", ] -[[package]] -name = "arrow-array" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" -dependencies = [ - "ahash", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", - "chrono", - "chrono-tz", - "half", - "hashbrown 0.16.1", - "num", -] - [[package]] name = "arrow-array" version = "57.1.0" @@ -299,9 +247,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eaff85a44e9fa914660fb0d0bb00b79c4a3d888b5334adb3ea4330c84f002" dependencies = [ "ahash", - "arrow-buffer 57.1.0", - "arrow-data 57.1.0", - "arrow-schema 57.1.0", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "chrono-tz", "half", @@ -311,17 +259,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "arrow-buffer" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" -dependencies = [ - "bytes", - "half", - "num", -] - [[package]] name = "arrow-buffer" version = "57.1.0" @@ -334,39 +271,18 @@ dependencies = [ "num-traits", ] -[[package]] -name = "arrow-cast" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" -dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", - "arrow-select 56.2.0", - "atoi", - "base64", - "chrono", - "comfy-table", - "half", - "lexical-core", - "num", - "ryu", -] - [[package]] name = "arrow-cast" version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3d131abb183f80c450d4591dc784f8d7750c50c6e2bc3fcaad148afc8361271" dependencies = [ - "arrow-array 57.1.0", - "arrow-buffer 57.1.0", - "arrow-data 57.1.0", - "arrow-ord 57.1.0", - "arrow-schema 57.1.0", - "arrow-select 57.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", "atoi", "base64", "chrono", @@ -377,173 +293,81 @@ dependencies = [ "ryu", ] -[[package]] -name = "arrow-csv" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa9bf02705b5cf762b6f764c65f04ae9082c7cfc4e96e0c33548ee3f67012eb" -dependencies = [ - "arrow-array 56.2.0", - "arrow-cast 56.2.0", - "arrow-schema 56.2.0", - "chrono", - "csv", - "csv-core", - "regex", -] - [[package]] name = "arrow-csv" version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2275877a0e5e7e7c76954669366c2aa1a829e340ab1f612e647507860906fb6b" dependencies = [ - "arrow-array 57.1.0", - "arrow-cast 57.1.0", - "arrow-schema 57.1.0", + "arrow-array", + "arrow-cast", + "arrow-schema", "chrono", "csv", "csv-core", "regex", ] -[[package]] -name = "arrow-data" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" -dependencies = [ - "arrow-buffer 56.2.0", - "arrow-schema 56.2.0", - "half", - "num", -] - [[package]] name = "arrow-data" version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05738f3d42cb922b9096f7786f606fcb8669260c2640df8490533bb2fa38c9d3" dependencies = [ - "arrow-buffer 57.1.0", - "arrow-schema 57.1.0", + "arrow-buffer", + "arrow-schema", "half", "num-integer", "num-traits", ] -[[package]] -name = "arrow-flight" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c8b0ba0784d56bc6266b79f5de7a24b47024e7b3a0045d2ad4df3d9b686099f" -dependencies = [ - "arrow-arith 56.2.0", - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-cast 56.2.0", - "arrow-data 56.2.0", - "arrow-ipc 56.2.0", - "arrow-ord 56.2.0", - "arrow-row 56.2.0", - "arrow-schema 56.2.0", - "arrow-select 56.2.0", - "arrow-string 56.2.0", - "base64", - "bytes", - "futures", - "once_cell", - "paste", - "prost 0.13.5", - "prost-types 0.13.5", - "tonic 0.13.1", -] - [[package]] name = "arrow-flight" version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b5f57c3d39d1b1b7c1376a772ea86a131e7da310aed54ebea9363124bb885e3" dependencies = [ - "arrow-array 57.1.0", - "arrow-buffer 57.1.0", - "arrow-cast 57.1.0", - "arrow-ipc 57.1.0", - "arrow-schema 57.1.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ipc", + "arrow-schema", "base64", "bytes", "futures", - "prost 0.14.1", - "prost-types 0.14.1", - "tonic 0.14.2", + "prost", + "prost-types", + "tonic", "tonic-prost", ] -[[package]] -name = "arrow-ipc" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d3594dcddccc7f20fd069bc8e9828ce37220372680ff638c5e00dea427d88f5" -dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", - "arrow-select 56.2.0", - "flatbuffers", - "lz4_flex 0.11.5", - "zstd", -] - [[package]] name = "arrow-ipc" version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d09446e8076c4b3f235603d9ea7c5494e73d441b01cd61fb33d7254c11964b3" dependencies = [ - "arrow-array 57.1.0", - "arrow-buffer 57.1.0", - "arrow-data 57.1.0", - "arrow-schema 57.1.0", - "arrow-select 57.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "flatbuffers", - "lz4_flex 0.12.0", + "lz4_flex", "zstd", ] -[[package]] -name = "arrow-json" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88cf36502b64a127dc659e3b305f1d993a544eab0d48cce704424e62074dc04b" -dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-cast 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", - "chrono", - "half", - "indexmap", - "lexical-core", - "memchr", - "num", - "serde", - "serde_json", - "simdutf8", -] - [[package]] name = "arrow-json" version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "371ffd66fa77f71d7628c63f209c9ca5341081051aa32f9c8020feb0def787c0" dependencies = [ - "arrow-array 57.1.0", - "arrow-buffer 57.1.0", - "arrow-cast 57.1.0", - "arrow-data 57.1.0", - "arrow-schema 57.1.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", "chrono", "half", "indexmap", @@ -557,43 +381,17 @@ dependencies = [ "simdutf8", ] -[[package]] -name = "arrow-ord" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" -dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", - "arrow-select 56.2.0", -] - [[package]] name = "arrow-ord" version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbc94fc7adec5d1ba9e8cd1b1e8d6f72423b33fe978bf1f46d970fafab787521" dependencies = [ - "arrow-array 57.1.0", - "arrow-buffer 57.1.0", - "arrow-data 57.1.0", - "arrow-schema 57.1.0", - "arrow-select 57.1.0", -] - -[[package]] -name = "arrow-row" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753" -dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", - "half", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", ] [[package]] @@ -602,23 +400,13 @@ version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "169676f317157dc079cc5def6354d16db63d8861d61046d2f3883268ced6f99f" dependencies = [ - "arrow-array 57.1.0", - "arrow-buffer 57.1.0", - "arrow-data 57.1.0", - "arrow-schema 57.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "half", ] -[[package]] -name = "arrow-schema" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "arrow-schema" version = "57.1.0" @@ -629,20 +417,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "arrow-select" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" -dependencies = [ - "ahash", - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", - "num", -] - [[package]] name = "arrow-select" version = "57.1.0" @@ -650,41 +424,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae980d021879ea119dd6e2a13912d81e64abed372d53163e804dfe84639d8010" dependencies = [ "ahash", - "arrow-array 57.1.0", - "arrow-buffer 57.1.0", - "arrow-data 57.1.0", - "arrow-schema 57.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "num-traits", ] -[[package]] -name = "arrow-string" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" -dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", - "arrow-select 56.2.0", - "memchr", - "num", - "regex", - "regex-syntax", -] - [[package]] name = "arrow-string" version = "57.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf35e8ef49dcf0c5f6d175edee6b8af7b45611805333129c541a8b89a0fc0534" dependencies = [ - "arrow-array 57.1.0", - "arrow-buffer 57.1.0", - "arrow-data 57.1.0", - "arrow-schema 57.1.0", - "arrow-select 57.1.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "memchr", "num-traits", "regex", @@ -1212,13 +969,10 @@ checksum = "5b098575ebe77cb6d14fc7f32749631a6e44edbef6b796f89b020e99ba20d425" dependencies = [ "axum-core 0.5.5", "bytes", - "form_urlencoded", "futures-util", "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.8.1", - "hyper-util", "itoa", "matchit 0.8.4", "memchr", @@ -1226,15 +980,10 @@ dependencies = [ "percent-encoding", "pin-project-lite", "serde_core", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", "sync_wrapper", - "tokio", "tower", "tower-layer", "tower-service", - "tracing", ] [[package]] @@ -1274,7 +1023,6 @@ dependencies = [ "sync_wrapper", "tower-layer", "tower-service", - "tracing", ] [[package]] @@ -1292,118 +1040,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "ballista" -version = "50.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f2a3159e4138cf7de0ba8b7267aa97eb0c01d960920b1747dba71e6986ef421" -dependencies = [ - "async-trait", - "ballista-core", - "ballista-executor", - "ballista-scheduler", - "datafusion 50.3.0", - "log", - "tokio", - "url", -] - -[[package]] -name = "ballista-core" -version = "50.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca9c994b0f51d52a219fbaf4ca1769f807795a998e9d3c50a03ced67693860e" -dependencies = [ - "arrow-flight 56.2.0", - "async-trait", - "aws-config", - "aws-credential-types", - "chrono", - "clap 4.5.53", - "datafusion 50.3.0", - "datafusion-proto 50.3.0", - "datafusion-proto-common 50.3.0", - "futures", - "itertools 0.14.0", - "log", - "md-5", - "object_store", - "parking_lot", - "prost 0.13.5", - "prost-types 0.13.5", - "rand 0.9.2", - "rustc_version", - "serde", - "tokio", - "tokio-stream", - "tonic 0.13.1", - "tonic-build", - "url", - "uuid", -] - -[[package]] -name = "ballista-executor" -version = "50.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1bf32e64828067a003f201fc0068da4863867936a680af6ca3af7edda94d6e" -dependencies = [ - "arrow 56.2.0", - "arrow-flight 56.2.0", - "async-trait", - "ballista-core", - "clap 4.5.53", - "dashmap", - "datafusion 50.3.0", - "datafusion-proto 50.3.0", - "futures", - "libc", - "log", - "mimalloc", - "parking_lot", - "tempfile", - "tokio", - "tokio-stream", - "tokio-util", - "tonic 0.13.1", - "tracing", - "tracing-appender", - "tracing-subscriber", - "uuid", -] - -[[package]] -name = "ballista-scheduler" -version = "50.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54e618c0dbe63c48f69b417b26a2824a6be849bbf78fd73e7d98b567ce5907f3" -dependencies = [ - "arrow-flight 56.2.0", - "async-trait", - "axum 0.8.7", - "ballista-core", - "clap 4.5.53", - "dashmap", - "datafusion 50.3.0", - "datafusion-proto 50.3.0", - "futures", - "http 1.3.1", - "log", - "object_store", - "parking_lot", - "prost 0.13.5", - "prost-types 0.13.5", - "rand 0.9.2", - "serde", - "tokio", - "tokio-stream", - "tonic 0.13.1", - "tracing", - "tracing-appender", - "tracing-subscriber", - "uuid", -] - [[package]] name = "base64" version = "0.22.1" @@ -2094,110 +1730,55 @@ dependencies = [ "parking_lot_core", ] -[[package]] -name = "datafusion" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af15bb3c6ffa33011ef579f6b0bcbe7c26584688bd6c994f548e44df67f011a" -dependencies = [ - "arrow 56.2.0", - "arrow-ipc 56.2.0", - "arrow-schema 56.2.0", - "async-trait", - "bytes", - "bzip2 0.6.1", - "chrono", - "datafusion-catalog 50.3.0", - "datafusion-catalog-listing 50.3.0", - "datafusion-common 50.3.0", - "datafusion-common-runtime 50.3.0", - "datafusion-datasource 50.3.0", - "datafusion-datasource-csv 50.3.0", - "datafusion-datasource-json 50.3.0", - "datafusion-datasource-parquet 50.3.0", - "datafusion-execution 50.3.0", - "datafusion-expr 50.3.0", - "datafusion-expr-common 50.3.0", - "datafusion-functions 50.3.0", - "datafusion-functions-aggregate 50.3.0", - "datafusion-functions-nested 50.3.0", - "datafusion-functions-table 50.3.0", - "datafusion-functions-window 50.3.0", - "datafusion-optimizer 50.3.0", - "datafusion-physical-expr 50.3.0", - "datafusion-physical-expr-adapter 50.3.0", - "datafusion-physical-expr-common 50.3.0", - "datafusion-physical-optimizer 50.3.0", - "datafusion-physical-plan 50.3.0", - "datafusion-session 50.3.0", - "datafusion-sql 50.3.0", - "flate2", - "futures", - "itertools 0.14.0", - "log", - "object_store", - "parking_lot", - "parquet 56.2.0", - "rand 0.9.2", - "regex", - "sqlparser 0.58.0", - "tempfile", - "tokio", - "url", - "uuid", - "xz2", - "zstd", -] - [[package]] name = "datafusion" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ba7cb113e9c0bedf9e9765926031e132fa05a1b09ba6e93a6d1a4d7044457b8" dependencies = [ - "arrow 57.1.0", - "arrow-schema 57.1.0", + "arrow", + "arrow-schema", "async-trait", "bytes", "bzip2 0.6.1", "chrono", - "datafusion-catalog 51.0.0", - "datafusion-catalog-listing 51.0.0", - "datafusion-common 51.0.0", - "datafusion-common-runtime 51.0.0", - "datafusion-datasource 51.0.0", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", "datafusion-datasource-arrow", "datafusion-datasource-avro", - "datafusion-datasource-csv 51.0.0", - "datafusion-datasource-json 51.0.0", - "datafusion-datasource-parquet 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-expr-common 51.0.0", - "datafusion-functions 51.0.0", - "datafusion-functions-aggregate 51.0.0", - "datafusion-functions-nested 51.0.0", - "datafusion-functions-table 51.0.0", - "datafusion-functions-window 51.0.0", - "datafusion-optimizer 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-expr-adapter 51.0.0", - "datafusion-physical-expr-common 51.0.0", - "datafusion-physical-optimizer 51.0.0", - "datafusion-physical-plan 51.0.0", - "datafusion-session 51.0.0", - "datafusion-sql 51.0.0", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", + "datafusion-functions-table", + "datafusion-functions-window", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-session", + "datafusion-sql", "flate2", "futures", "itertools 0.14.0", "log", "object_store", "parking_lot", - "parquet 57.1.0", + "parquet", "rand 0.9.2", "regex", "rstest", - "sqlparser 0.59.0", + "sqlparser", "tempfile", "tokio", "url", @@ -2208,22 +1789,21 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "50.3.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187622262ad8f7d16d3be9202b4c1e0116f1c9aa387e5074245538b755261621" +checksum = "66a3a799f914a59b1ea343906a0486f17061f39509af74e874a866428951130d" dependencies = [ - "arrow 56.2.0", + "arrow", "async-trait", "dashmap", - "datafusion-common 50.3.0", - "datafusion-common-runtime 50.3.0", - "datafusion-datasource 50.3.0", - "datafusion-execution 50.3.0", - "datafusion-expr 50.3.0", - "datafusion-physical-expr 50.3.0", - "datafusion-physical-plan 50.3.0", - "datafusion-session 50.3.0", - "datafusion-sql 50.3.0", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-session", "futures", "itertools 0.14.0", "log", @@ -2233,130 +1813,57 @@ dependencies = [ ] [[package]] -name = "datafusion-catalog" +name = "datafusion-catalog-listing" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a3a799f914a59b1ea343906a0486f17061f39509af74e874a866428951130d" +checksum = "6db1b113c80d7a0febcd901476a57aef378e717c54517a163ed51417d87621b0" dependencies = [ - "arrow 57.1.0", + "arrow", "async-trait", - "dashmap", - "datafusion-common 51.0.0", - "datafusion-common-runtime 51.0.0", - "datafusion-datasource 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-plan 51.0.0", - "datafusion-session 51.0.0", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", "futures", "itertools 0.14.0", "log", "object_store", - "parking_lot", "tokio", ] [[package]] -name = "datafusion-catalog-listing" -version = "50.3.0" +name = "datafusion-cli" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9657314f0a32efd0382b9a46fdeb2d233273ece64baa68a7c45f5a192daf0f83" +checksum = "fab982df44f818a749cb5200504ccb919f4608cb9808daf8b3fb98aa7955fd1e" dependencies = [ - "arrow 56.2.0", + "arrow", "async-trait", - "datafusion-catalog 50.3.0", - "datafusion-common 50.3.0", - "datafusion-datasource 50.3.0", - "datafusion-execution 50.3.0", - "datafusion-expr 50.3.0", - "datafusion-physical-expr 50.3.0", - "datafusion-physical-expr-common 50.3.0", - "datafusion-physical-plan 50.3.0", - "datafusion-session 50.3.0", + "aws-config", + "aws-credential-types", + "chrono", + "clap 4.5.53", + "datafusion", + "datafusion-common", + "dirs", + "env_logger", "futures", "log", - "object_store", - "tokio", -] - -[[package]] -name = "datafusion-catalog-listing" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db1b113c80d7a0febcd901476a57aef378e717c54517a163ed51417d87621b0" -dependencies = [ - "arrow 57.1.0", - "async-trait", - "datafusion-catalog 51.0.0", - "datafusion-common 51.0.0", - "datafusion-datasource 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-expr-adapter 51.0.0", - "datafusion-physical-expr-common 51.0.0", - "datafusion-physical-plan 51.0.0", - "futures", - "itertools 0.14.0", - "log", - "object_store", - "tokio", -] - -[[package]] -name = "datafusion-cli" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fab982df44f818a749cb5200504ccb919f4608cb9808daf8b3fb98aa7955fd1e" -dependencies = [ - "arrow 57.1.0", - "async-trait", - "aws-config", - "aws-credential-types", - "chrono", - "clap 4.5.53", - "datafusion 51.0.0", - "datafusion-common 51.0.0", - "dirs", - "env_logger", - "futures", - "log", - "mimalloc", + "mimalloc", "object_store", "parking_lot", - "parquet 57.1.0", + "parquet", "regex", "rustyline", "tokio", "url", ] -[[package]] -name = "datafusion-common" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a83760d9a13122d025fbdb1d5d5aaf93dd9ada5e90ea229add92aa30898b2d1" -dependencies = [ - "ahash", - "arrow 56.2.0", - "arrow-ipc 56.2.0", - "base64", - "chrono", - "half", - "hashbrown 0.14.5", - "indexmap", - "libc", - "log", - "object_store", - "parquet 56.2.0", - "paste", - "recursive", - "sqlparser 0.58.0", - "tokio", - "web-time", -] - [[package]] name = "datafusion-common" version = "51.0.0" @@ -2365,8 +1872,8 @@ checksum = "7c10f7659e96127d25e8366be7c8be4109595d6a2c3eac70421f380a7006a1b0" dependencies = [ "ahash", "apache-avro", - "arrow 57.1.0", - "arrow-ipc 57.1.0", + "arrow", + "arrow-ipc", "chrono", "half", "hashbrown 0.14.5", @@ -2375,25 +1882,14 @@ dependencies = [ "libc", "log", "object_store", - "parquet 57.1.0", + "parquet", "paste", "recursive", - "sqlparser 0.59.0", + "sqlparser", "tokio", "web-time", ] -[[package]] -name = "datafusion-common-runtime" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b6234a6c7173fe5db1c6c35c01a12b2aa0f803a3007feee53483218817f8b1e" -dependencies = [ - "futures", - "log", - "tokio", -] - [[package]] name = "datafusion-common-runtime" version = "51.0.0" @@ -2405,64 +1901,27 @@ dependencies = [ "tokio", ] -[[package]] -name = "datafusion-datasource" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7256c9cb27a78709dd42d0c80f0178494637209cac6e29d5c93edd09b6721b86" -dependencies = [ - "arrow 56.2.0", - "async-compression", - "async-trait", - "bytes", - "bzip2 0.6.1", - "chrono", - "datafusion-common 50.3.0", - "datafusion-common-runtime 50.3.0", - "datafusion-execution 50.3.0", - "datafusion-expr 50.3.0", - "datafusion-physical-expr 50.3.0", - "datafusion-physical-expr-adapter 50.3.0", - "datafusion-physical-expr-common 50.3.0", - "datafusion-physical-plan 50.3.0", - "datafusion-session 50.3.0", - "flate2", - "futures", - "glob", - "itertools 0.14.0", - "log", - "object_store", - "parquet 56.2.0", - "rand 0.9.2", - "tempfile", - "tokio", - "tokio-util", - "url", - "xz2", - "zstd", -] - [[package]] name = "datafusion-datasource" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fde13794244bc7581cd82f6fff217068ed79cdc344cafe4ab2c3a1c3510b38d6" dependencies = [ - "arrow 57.1.0", + "arrow", "async-compression", "async-trait", "bytes", "bzip2 0.6.1", "chrono", - "datafusion-common 51.0.0", - "datafusion-common-runtime 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-expr-adapter 51.0.0", - "datafusion-physical-expr-common 51.0.0", - "datafusion-physical-plan 51.0.0", - "datafusion-session 51.0.0", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", "flate2", "futures", "glob", @@ -2483,18 +1942,18 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "804fa9b4ecf3157982021770617200ef7c1b2979d57bec9044748314775a9aea" dependencies = [ - "arrow 57.1.0", - "arrow-ipc 57.1.0", + "arrow", + "arrow-ipc", "async-trait", "bytes", - "datafusion-common 51.0.0", - "datafusion-common-runtime 51.0.0", - "datafusion-datasource 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-physical-expr-common 51.0.0", - "datafusion-physical-plan 51.0.0", - "datafusion-session 51.0.0", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", "futures", "itertools 0.14.0", "object_store", @@ -2508,144 +1967,61 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388ed8be535f562cc655b9c3d22edbfb0f1a50a25c242647a98b6d92a75b55a1" dependencies = [ "apache-avro", - "arrow 57.1.0", + "arrow", "async-trait", "bytes", - "datafusion-common 51.0.0", - "datafusion-datasource 51.0.0", - "datafusion-physical-expr-common 51.0.0", - "datafusion-physical-plan 51.0.0", - "datafusion-session 51.0.0", + "datafusion-common", + "datafusion-datasource", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", "futures", "num-traits", "object_store", ] -[[package]] -name = "datafusion-datasource-csv" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64533a90f78e1684bfb113d200b540f18f268134622d7c96bbebc91354d04825" -dependencies = [ - "arrow 56.2.0", - "async-trait", - "bytes", - "datafusion-catalog 50.3.0", - "datafusion-common 50.3.0", - "datafusion-common-runtime 50.3.0", - "datafusion-datasource 50.3.0", - "datafusion-execution 50.3.0", - "datafusion-expr 50.3.0", - "datafusion-physical-expr 50.3.0", - "datafusion-physical-expr-common 50.3.0", - "datafusion-physical-plan 50.3.0", - "datafusion-session 50.3.0", - "futures", - "object_store", - "regex", - "tokio", -] - [[package]] name = "datafusion-datasource-csv" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61a1641a40b259bab38131c5e6f48fac0717bedb7dc93690e604142a849e0568" dependencies = [ - "arrow 57.1.0", + "arrow", "async-trait", "bytes", - "datafusion-common 51.0.0", - "datafusion-common-runtime 51.0.0", - "datafusion-datasource 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-physical-expr-common 51.0.0", - "datafusion-physical-plan 51.0.0", - "datafusion-session 51.0.0", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", "futures", "object_store", "regex", "tokio", ] -[[package]] -name = "datafusion-datasource-json" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d7ebeb12c77df0aacad26f21b0d033aeede423a64b2b352f53048a75bf1d6e6" -dependencies = [ - "arrow 56.2.0", - "async-trait", - "bytes", - "datafusion-catalog 50.3.0", - "datafusion-common 50.3.0", - "datafusion-common-runtime 50.3.0", - "datafusion-datasource 50.3.0", - "datafusion-execution 50.3.0", - "datafusion-expr 50.3.0", - "datafusion-physical-expr 50.3.0", - "datafusion-physical-expr-common 50.3.0", - "datafusion-physical-plan 50.3.0", - "datafusion-session 50.3.0", - "futures", - "object_store", - "serde_json", - "tokio", -] - [[package]] name = "datafusion-datasource-json" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adeacdb00c1d37271176f8fb6a1d8ce096baba16ea7a4b2671840c5c9c64fe85" dependencies = [ - "arrow 57.1.0", - "async-trait", - "bytes", - "datafusion-common 51.0.0", - "datafusion-common-runtime 51.0.0", - "datafusion-datasource 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-physical-expr-common 51.0.0", - "datafusion-physical-plan 51.0.0", - "datafusion-session 51.0.0", - "futures", - "object_store", - "tokio", -] - -[[package]] -name = "datafusion-datasource-parquet" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09e783c4c7d7faa1199af2df4761c68530634521b176a8d1331ddbc5a5c75133" -dependencies = [ - "arrow 56.2.0", + "arrow", "async-trait", "bytes", - "datafusion-catalog 50.3.0", - "datafusion-common 50.3.0", - "datafusion-common-runtime 50.3.0", - "datafusion-datasource 50.3.0", - "datafusion-execution 50.3.0", - "datafusion-expr 50.3.0", - "datafusion-functions-aggregate 50.3.0", - "datafusion-physical-expr 50.3.0", - "datafusion-physical-expr-adapter 50.3.0", - "datafusion-physical-expr-common 50.3.0", - "datafusion-physical-optimizer 50.3.0", - "datafusion-physical-plan 50.3.0", - "datafusion-pruning 50.3.0", - "datafusion-session 50.3.0", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", "futures", - "itertools 0.14.0", - "log", "object_store", - "parking_lot", - "parquet 56.2.0", - "rand 0.9.2", "tokio", ] @@ -2655,27 +2031,27 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43d0b60ffd66f28bfb026565d62b0a6cbc416da09814766a3797bba7d85a3cd9" dependencies = [ - "arrow 57.1.0", + "arrow", "async-trait", "bytes", - "datafusion-common 51.0.0", - "datafusion-common-runtime 51.0.0", - "datafusion-datasource 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-functions-aggregate-common 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-expr-adapter 51.0.0", - "datafusion-physical-expr-common 51.0.0", - "datafusion-physical-plan 51.0.0", - "datafusion-pruning 51.0.0", - "datafusion-session 51.0.0", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", + "datafusion-session", "futures", "itertools 0.14.0", "log", "object_store", "parking_lot", - "parquet 57.1.0", + "parquet", "tokio", ] @@ -2683,15 +2059,15 @@ dependencies = [ name = "datafusion-distributed" version = "0.1.0" dependencies = [ - "arrow 57.1.0", - "arrow-flight 57.1.0", - "arrow-select 57.1.0", + "arrow", + "arrow-flight", + "arrow-select", "async-trait", "bytes", "chrono", "dashmap", - "datafusion 51.0.0", - "datafusion-proto 51.0.0", + "datafusion", + "datafusion-proto", "delegate", "futures", "http 1.3.1", @@ -2700,16 +2076,16 @@ dependencies = [ "itertools 0.14.0", "moka", "object_store", - "parquet 57.1.0", + "parquet", "pin-project", "pretty_assertions", - "prost 0.14.1", + "prost", "rand 0.8.5", "reqwest", "structopt", "tokio", "tokio-stream", - "tonic 0.14.2", + "tonic", "tower", "tpchgen", "tpchgen-arrow", @@ -2722,33 +2098,29 @@ dependencies = [ name = "datafusion-distributed-benchmarks" version = "0.1.0" dependencies = [ - "arrow-flight 57.1.0", + "arrow-flight", "async-trait", "aws-config", "aws-sdk-ec2", "axum 0.7.9", - "ballista", - "ballista-core", - "ballista-executor", - "ballista-scheduler", "chrono", "clap 4.5.53", "dashmap", - "datafusion 51.0.0", + "datafusion", "datafusion-distributed", - "datafusion-proto 51.0.0", + "datafusion-proto", "env_logger", "futures", "log", "object_store", "openssl", - "parquet 57.1.0", - "prost 0.14.1", + "parquet", + "prost", "serde", "serde_json", "structopt", "tokio", - "tonic 0.14.2", + "tonic", "url", ] @@ -2756,10 +2128,10 @@ dependencies = [ name = "datafusion-distributed-cli" version = "0.1.0" dependencies = [ - "arrow-flight 57.1.0", + "arrow-flight", "async-trait", "clap 4.5.53", - "datafusion 51.0.0", + "datafusion", "datafusion-cli", "datafusion-distributed", "dirs", @@ -2767,120 +2139,59 @@ dependencies = [ "hyper-util", "tokio", "tokio-stream", - "tonic 0.14.2", + "tonic", "tower", "url", ] -[[package]] -name = "datafusion-doc" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99ee6b1d9a80d13f9deb2291f45c07044b8e62fb540dbde2453a18be17a36429" - [[package]] name = "datafusion-doc" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b99e13947667b36ad713549237362afb054b2d8f8cc447751e23ec61202db07" -[[package]] -name = "datafusion-execution" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4cec0a57653bec7b933fb248d3ffa3fa3ab3bd33bd140dc917f714ac036f531" -dependencies = [ - "arrow 56.2.0", - "async-trait", - "dashmap", - "datafusion-common 50.3.0", - "datafusion-expr 50.3.0", - "futures", - "log", - "object_store", - "parking_lot", - "rand 0.9.2", - "tempfile", - "url", -] - [[package]] name = "datafusion-execution" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63695643190679037bc946ad46a263b62016931547bf119859c511f7ff2f5178" dependencies = [ - "arrow 57.1.0", + "arrow", "async-trait", "dashmap", - "datafusion-common 51.0.0", - "datafusion-expr 51.0.0", + "datafusion-common", + "datafusion-expr", "futures", "log", "object_store", "parking_lot", - "parquet 57.1.0", + "parquet", "rand 0.9.2", "tempfile", "url", ] -[[package]] -name = "datafusion-expr" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef76910bdca909722586389156d0aa4da4020e1631994d50fadd8ad4b1aa05fe" -dependencies = [ - "arrow 56.2.0", - "async-trait", - "chrono", - "datafusion-common 50.3.0", - "datafusion-doc 50.3.0", - "datafusion-expr-common 50.3.0", - "datafusion-functions-aggregate-common 50.3.0", - "datafusion-functions-window-common 50.3.0", - "datafusion-physical-expr-common 50.3.0", - "indexmap", - "paste", - "recursive", - "serde_json", - "sqlparser 0.58.0", -] - [[package]] name = "datafusion-expr" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9a4787cbf5feb1ab351f789063398f67654a6df75c4d37d7f637dc96f951a91" dependencies = [ - "arrow 57.1.0", + "arrow", "async-trait", "chrono", - "datafusion-common 51.0.0", - "datafusion-doc 51.0.0", - "datafusion-expr-common 51.0.0", - "datafusion-functions-aggregate-common 51.0.0", - "datafusion-functions-window-common 51.0.0", - "datafusion-physical-expr-common 51.0.0", + "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", "indexmap", "itertools 0.14.0", "paste", "recursive", "serde_json", - "sqlparser 0.59.0", -] - -[[package]] -name = "datafusion-expr-common" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d155ccbda29591ca71a1344dd6bed26c65a4438072b400df9db59447f590bb6" -dependencies = [ - "arrow 56.2.0", - "datafusion-common 50.3.0", - "indexmap", - "itertools 0.14.0", - "paste", + "sqlparser", ] [[package]] @@ -2889,60 +2200,31 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ce2fb1b8c15c9ac45b0863c30b268c69dc9ee7a1ee13ecf5d067738338173dc" dependencies = [ - "arrow 57.1.0", - "datafusion-common 51.0.0", + "arrow", + "datafusion-common", "indexmap", "itertools 0.14.0", "paste", ] -[[package]] -name = "datafusion-functions" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de2782136bd6014670fd84fe3b0ca3b3e4106c96403c3ae05c0598577139977" -dependencies = [ - "arrow 56.2.0", - "arrow-buffer 56.2.0", - "base64", - "blake2", - "blake3", - "chrono", - "datafusion-common 50.3.0", - "datafusion-doc 50.3.0", - "datafusion-execution 50.3.0", - "datafusion-expr 50.3.0", - "datafusion-expr-common 50.3.0", - "datafusion-macros 50.3.0", - "hex", - "itertools 0.14.0", - "log", - "md-5", - "rand 0.9.2", - "regex", - "sha2", - "unicode-segmentation", - "uuid", -] - [[package]] name = "datafusion-functions" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "794a9db7f7b96b3346fc007ff25e994f09b8f0511b4cf7dff651fadfe3ebb28f" dependencies = [ - "arrow 57.1.0", - "arrow-buffer 57.1.0", + "arrow", + "arrow-buffer", "base64", "blake2", "blake3", "chrono", - "datafusion-common 51.0.0", - "datafusion-doc 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-expr-common 51.0.0", - "datafusion-macros 51.0.0", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", "hex", "itertools 0.14.0", "log", @@ -2955,27 +2237,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "datafusion-functions-aggregate" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07331fc13603a9da97b74fd8a273f4238222943dffdbbed1c4c6f862a30105bf" -dependencies = [ - "ahash", - "arrow 56.2.0", - "datafusion-common 50.3.0", - "datafusion-doc 50.3.0", - "datafusion-execution 50.3.0", - "datafusion-expr 50.3.0", - "datafusion-functions-aggregate-common 50.3.0", - "datafusion-macros 50.3.0", - "datafusion-physical-expr 50.3.0", - "datafusion-physical-expr-common 50.3.0", - "half", - "log", - "paste", -] - [[package]] name = "datafusion-functions-aggregate" version = "51.0.0" @@ -2983,33 +2244,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c25210520a9dcf9c2b2cbbce31ebd4131ef5af7fc60ee92b266dc7d159cb305" dependencies = [ "ahash", - "arrow 57.1.0", - "datafusion-common 51.0.0", - "datafusion-doc 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-functions-aggregate-common 51.0.0", - "datafusion-macros 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-expr-common 51.0.0", + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "half", "log", "paste", ] -[[package]] -name = "datafusion-functions-aggregate-common" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5951e572a8610b89968a09b5420515a121fbc305c0258651f318dc07c97ab17" -dependencies = [ - "ahash", - "arrow 56.2.0", - "datafusion-common 50.3.0", - "datafusion-expr-common 50.3.0", - "datafusion-physical-expr-common 50.3.0", -] - [[package]] name = "datafusion-functions-aggregate-common" version = "51.0.0" @@ -3017,32 +2265,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62f4a66f3b87300bb70f4124b55434d2ae3fe80455f3574701d0348da040b55d" dependencies = [ "ahash", - "arrow 57.1.0", - "datafusion-common 51.0.0", - "datafusion-expr-common 51.0.0", - "datafusion-physical-expr-common 51.0.0", -] - -[[package]] -name = "datafusion-functions-nested" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdacca9302c3d8fc03f3e94f338767e786a88a33f5ebad6ffc0e7b50364b9ea3" -dependencies = [ - "arrow 56.2.0", - "arrow-ord 56.2.0", - "datafusion-common 50.3.0", - "datafusion-doc 50.3.0", - "datafusion-execution 50.3.0", - "datafusion-expr 50.3.0", - "datafusion-functions 50.3.0", - "datafusion-functions-aggregate 50.3.0", - "datafusion-functions-aggregate-common 50.3.0", - "datafusion-macros 50.3.0", - "datafusion-physical-expr-common 50.3.0", - "itertools 0.14.0", - "log", - "paste", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", ] [[package]] @@ -3051,120 +2277,65 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae5c06eed03918dc7fe7a9f082a284050f0e9ecf95d72f57712d1496da03b8c4" dependencies = [ - "arrow 57.1.0", - "arrow-ord 57.1.0", - "datafusion-common 51.0.0", - "datafusion-doc 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-expr-common 51.0.0", - "datafusion-functions 51.0.0", - "datafusion-functions-aggregate 51.0.0", - "datafusion-functions-aggregate-common 51.0.0", - "datafusion-macros 51.0.0", - "datafusion-physical-expr-common 51.0.0", + "arrow", + "arrow-ord", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr-common", "itertools 0.14.0", "log", "paste", ] -[[package]] -name = "datafusion-functions-table" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c37ff8a99434fbbad604a7e0669717c58c7c4f14c472d45067c4b016621d981" -dependencies = [ - "arrow 56.2.0", - "async-trait", - "datafusion-catalog 50.3.0", - "datafusion-common 50.3.0", - "datafusion-expr 50.3.0", - "datafusion-physical-plan 50.3.0", - "parking_lot", - "paste", -] - [[package]] name = "datafusion-functions-table" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db4fed1d71738fbe22e2712d71396db04c25de4111f1ec252b8f4c6d3b25d7f5" dependencies = [ - "arrow 57.1.0", + "arrow", "async-trait", - "datafusion-catalog 51.0.0", - "datafusion-common 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-physical-plan 51.0.0", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-plan", "parking_lot", "paste", ] -[[package]] -name = "datafusion-functions-window" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e2aea7c79c926cffabb13dc27309d4eaeb130f4a21c8ba91cdd241c813652b" -dependencies = [ - "arrow 56.2.0", - "datafusion-common 50.3.0", - "datafusion-doc 50.3.0", - "datafusion-expr 50.3.0", - "datafusion-functions-window-common 50.3.0", - "datafusion-macros 50.3.0", - "datafusion-physical-expr 50.3.0", - "datafusion-physical-expr-common 50.3.0", - "log", - "paste", -] - [[package]] name = "datafusion-functions-window" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d92206aa5ae21892f1552b4d61758a862a70956e6fd7a95cb85db1de74bc6d1" dependencies = [ - "arrow 57.1.0", - "datafusion-common 51.0.0", - "datafusion-doc 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-functions-window-common 51.0.0", - "datafusion-macros 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-expr-common 51.0.0", + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "log", "paste", ] -[[package]] -name = "datafusion-functions-window-common" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fead257ab5fd2ffc3b40fda64da307e20de0040fe43d49197241d9de82a487f" -dependencies = [ - "datafusion-common 50.3.0", - "datafusion-physical-expr-common 50.3.0", -] - [[package]] name = "datafusion-functions-window-common" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53ae9bcc39800820d53a22d758b3b8726ff84a5a3e24cecef04ef4e5fdf1c7cc" dependencies = [ - "datafusion-common 51.0.0", - "datafusion-physical-expr-common 51.0.0", -] - -[[package]] -name = "datafusion-macros" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec6f637bce95efac05cdfb9b6c19579ed4aa5f6b94d951cfa5bb054b7bb4f730" -dependencies = [ - "datafusion-expr 50.3.0", - "quote", - "syn 2.0.110", + "datafusion-common", + "datafusion-physical-expr-common", ] [[package]] @@ -3173,43 +2344,23 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1063ad4c9e094b3f798acee16d9a47bd7372d9699be2de21b05c3bd3f34ab848" dependencies = [ - "datafusion-doc 51.0.0", + "datafusion-doc", "quote", "syn 2.0.110", ] -[[package]] -name = "datafusion-optimizer" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6583ef666ae000a613a837e69e456681a9faa96347bf3877661e9e89e141d8a" -dependencies = [ - "arrow 56.2.0", - "chrono", - "datafusion-common 50.3.0", - "datafusion-expr 50.3.0", - "datafusion-expr-common 50.3.0", - "datafusion-physical-expr 50.3.0", - "indexmap", - "itertools 0.14.0", - "log", - "recursive", - "regex", - "regex-syntax", -] - [[package]] name = "datafusion-optimizer" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f35f9ec5d08b87fd1893a30c2929f2559c2f9806ca072d8fefca5009dc0f06a" dependencies = [ - "arrow 57.1.0", + "arrow", "chrono", - "datafusion-common 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-expr-common 51.0.0", - "datafusion-physical-expr 51.0.0", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", "indexmap", "itertools 0.14.0", "log", @@ -3218,29 +2369,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "datafusion-physical-expr" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8668103361a272cbbe3a61f72eca60c9b7c706e87cc3565bcf21e2b277b84f6" -dependencies = [ - "ahash", - "arrow 56.2.0", - "datafusion-common 50.3.0", - "datafusion-expr 50.3.0", - "datafusion-expr-common 50.3.0", - "datafusion-functions-aggregate-common 50.3.0", - "datafusion-physical-expr-common 50.3.0", - "half", - "hashbrown 0.14.5", - "indexmap", - "itertools 0.14.0", - "log", - "parking_lot", - "paste", - "petgraph 0.8.3", -] - [[package]] name = "datafusion-physical-expr" version = "51.0.0" @@ -3248,34 +2376,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c30cc8012e9eedcb48bbe112c6eff4ae5ed19cf3003cb0f505662e88b7014c5d" dependencies = [ "ahash", - "arrow 57.1.0", - "datafusion-common 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-expr-common 51.0.0", - "datafusion-functions-aggregate-common 51.0.0", - "datafusion-physical-expr-common 51.0.0", + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", "half", "hashbrown 0.14.5", "indexmap", "itertools 0.14.0", "parking_lot", "paste", - "petgraph 0.8.3", -] - -[[package]] -name = "datafusion-physical-expr-adapter" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "815acced725d30601b397e39958e0e55630e0a10d66ef7769c14ae6597298bb0" -dependencies = [ - "arrow 56.2.0", - "datafusion-common 50.3.0", - "datafusion-expr 50.3.0", - "datafusion-functions 50.3.0", - "datafusion-physical-expr 50.3.0", - "datafusion-physical-expr-common 50.3.0", - "itertools 0.14.0", + "petgraph", ] [[package]] @@ -3284,26 +2397,12 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f9ff2dbd476221b1f67337699eff432781c4e6e1713d2aefdaa517dfbf79768" dependencies = [ - "arrow 57.1.0", - "datafusion-common 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-functions 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-expr-common 51.0.0", - "itertools 0.14.0", -] - -[[package]] -name = "datafusion-physical-expr-common" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6652fe7b5bf87e85ed175f571745305565da2c0b599d98e697bcbedc7baa47c3" -dependencies = [ - "ahash", - "arrow 56.2.0", - "datafusion-common 50.3.0", - "datafusion-expr-common 50.3.0", - "hashbrown 0.14.5", + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "itertools 0.14.0", ] @@ -3314,81 +2413,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90da43e1ec550b172f34c87ec68161986ced70fd05c8d2a2add66eef9c276f03" dependencies = [ "ahash", - "arrow 57.1.0", - "datafusion-common 51.0.0", - "datafusion-expr-common 51.0.0", + "arrow", + "datafusion-common", + "datafusion-expr-common", "hashbrown 0.14.5", "itertools 0.14.0", ] [[package]] -name = "datafusion-physical-optimizer" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b7d623eb6162a3332b564a0907ba00895c505d101b99af78345f1acf929b5c" -dependencies = [ - "arrow 56.2.0", - "datafusion-common 50.3.0", - "datafusion-execution 50.3.0", - "datafusion-expr 50.3.0", - "datafusion-expr-common 50.3.0", - "datafusion-physical-expr 50.3.0", - "datafusion-physical-expr-common 50.3.0", - "datafusion-physical-plan 50.3.0", - "datafusion-pruning 50.3.0", - "itertools 0.14.0", - "log", - "recursive", -] - -[[package]] -name = "datafusion-physical-optimizer" -version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce9804f799acd7daef3be7aaffe77c0033768ed8fdbf5fb82fc4c5f2e6bc14e6" -dependencies = [ - "arrow 57.1.0", - "datafusion-common 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-expr-common 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-expr-common 51.0.0", - "datafusion-physical-plan 51.0.0", - "datafusion-pruning 51.0.0", - "itertools 0.14.0", - "recursive", -] - -[[package]] -name = "datafusion-physical-plan" -version = "50.3.0" +name = "datafusion-physical-optimizer" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2f7f778a1a838dec124efb96eae6144237d546945587557c9e6936b3414558c" +checksum = "ce9804f799acd7daef3be7aaffe77c0033768ed8fdbf5fb82fc4c5f2e6bc14e6" dependencies = [ - "ahash", - "arrow 56.2.0", - "arrow-ord 56.2.0", - "arrow-schema 56.2.0", - "async-trait", - "chrono", - "datafusion-common 50.3.0", - "datafusion-common-runtime 50.3.0", - "datafusion-execution 50.3.0", - "datafusion-expr 50.3.0", - "datafusion-functions-aggregate-common 50.3.0", - "datafusion-functions-window-common 50.3.0", - "datafusion-physical-expr 50.3.0", - "datafusion-physical-expr-common 50.3.0", - "futures", - "half", - "hashbrown 0.14.5", - "indexmap", + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", "itertools 0.14.0", - "log", - "parking_lot", - "pin-project-lite", - "tokio", + "recursive", ] [[package]] @@ -3398,19 +2446,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0acf0ad6b6924c6b1aa7d213b181e012e2d3ec0a64ff5b10ee6282ab0f8532ac" dependencies = [ "ahash", - "arrow 57.1.0", - "arrow-ord 57.1.0", - "arrow-schema 57.1.0", + "arrow", + "arrow-ord", + "arrow-schema", "async-trait", "chrono", - "datafusion-common 51.0.0", - "datafusion-common-runtime 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-functions-aggregate-common 51.0.0", - "datafusion-functions-window-common 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-expr-common 51.0.0", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "futures", "half", "hashbrown 0.14.5", @@ -3422,58 +2470,31 @@ dependencies = [ "tokio", ] -[[package]] -name = "datafusion-proto" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7df9f606892e6af45763d94d210634eec69b9bb6ced5353381682ff090028a3" -dependencies = [ - "arrow 56.2.0", - "chrono", - "datafusion 50.3.0", - "datafusion-common 50.3.0", - "datafusion-expr 50.3.0", - "datafusion-proto-common 50.3.0", - "object_store", - "prost 0.13.5", -] - [[package]] name = "datafusion-proto" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d368093a98a17d1449b1083ac22ed16b7128e4c67789991869480d8c4a40ecb9" dependencies = [ - "arrow 57.1.0", + "arrow", "chrono", - "datafusion-catalog 51.0.0", - "datafusion-catalog-listing 51.0.0", - "datafusion-common 51.0.0", - "datafusion-datasource 51.0.0", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-datasource", "datafusion-datasource-arrow", - "datafusion-datasource-csv 51.0.0", - "datafusion-datasource-json 51.0.0", - "datafusion-datasource-parquet 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-functions-table 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-expr-common 51.0.0", - "datafusion-physical-plan 51.0.0", - "datafusion-proto-common 51.0.0", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-table", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-common", "object_store", - "prost 0.14.1", -] - -[[package]] -name = "datafusion-proto-common" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4b14f288ca4ef77743d9672cafecf3adfffff0b9b04af9af79ecbeaaf736901" -dependencies = [ - "arrow 56.2.0", - "datafusion-common 50.3.0", - "prost 0.13.5", + "prost", ] [[package]] @@ -3482,27 +2503,9 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b6aef3d5e5c1d2bc3114c4876730cb76a9bdc5a8df31ef1b6db48f0c1671895" dependencies = [ - "arrow 57.1.0", - "datafusion-common 51.0.0", - "prost 0.14.1", -] - -[[package]] -name = "datafusion-pruning" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd1e59e2ca14fe3c30f141600b10ad8815e2856caa59ebbd0e3e07cd3d127a65" -dependencies = [ - "arrow 56.2.0", - "arrow-schema 56.2.0", - "datafusion-common 50.3.0", - "datafusion-datasource 50.3.0", - "datafusion-expr-common 50.3.0", - "datafusion-physical-expr 50.3.0", - "datafusion-physical-expr-common 50.3.0", - "datafusion-physical-plan 50.3.0", - "itertools 0.14.0", - "log", + "arrow", + "datafusion-common", + "prost", ] [[package]] @@ -3511,39 +2514,15 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac2c2498a1f134a9e11a9f5ed202a2a7d7e9774bd9249295593053ea3be999db" dependencies = [ - "arrow 57.1.0", - "datafusion-common 51.0.0", - "datafusion-datasource 51.0.0", - "datafusion-expr-common 51.0.0", - "datafusion-physical-expr 51.0.0", - "datafusion-physical-expr-common 51.0.0", - "datafusion-physical-plan 51.0.0", - "itertools 0.14.0", - "log", -] - -[[package]] -name = "datafusion-session" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21ef8e2745583619bd7a49474e8f45fbe98ebb31a133f27802217125a7b3d58d" -dependencies = [ - "arrow 56.2.0", - "async-trait", - "dashmap", - "datafusion-common 50.3.0", - "datafusion-common-runtime 50.3.0", - "datafusion-execution 50.3.0", - "datafusion-expr 50.3.0", - "datafusion-physical-expr 50.3.0", - "datafusion-physical-plan 50.3.0", - "datafusion-sql 50.3.0", - "futures", + "arrow", + "datafusion-common", + "datafusion-datasource", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", "itertools 0.14.0", "log", - "object_store", - "parking_lot", - "tokio", ] [[package]] @@ -3553,46 +2532,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f96eebd17555386f459037c65ab73aae8df09f464524c709d6a3134ad4f4776" dependencies = [ "async-trait", - "datafusion-common 51.0.0", - "datafusion-execution 51.0.0", - "datafusion-expr 51.0.0", - "datafusion-physical-plan 51.0.0", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", "parking_lot", ] -[[package]] -name = "datafusion-sql" -version = "50.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89abd9868770386fede29e5a4b14f49c0bf48d652c3b9d7a8a0332329b87d50b" -dependencies = [ - "arrow 56.2.0", - "bigdecimal", - "datafusion-common 50.3.0", - "datafusion-expr 50.3.0", - "indexmap", - "log", - "recursive", - "regex", - "sqlparser 0.58.0", -] - [[package]] name = "datafusion-sql" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fc195fe60634b2c6ccfd131b487de46dc30eccae8a3c35a13f136e7f440414f" dependencies = [ - "arrow 57.1.0", + "arrow", "bigdecimal", "chrono", - "datafusion-common 51.0.0", - "datafusion-expr 51.0.0", + "datafusion-common", + "datafusion-expr", "indexmap", "log", "recursive", "regex", - "sqlparser 0.59.0", + "sqlparser", ] [[package]] @@ -4897,15 +3859,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" -[[package]] -name = "lz4_flex" -version = "0.11.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" -dependencies = [ - "twox-hash", -] - [[package]] name = "lz4_flex" version = "0.12.0" @@ -4946,15 +3899,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "matchers" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" -dependencies = [ - "regex-automata", -] - [[package]] name = "matchit" version = "0.7.3" @@ -5058,12 +4002,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "multimap" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" - [[package]] name = "native-tls" version = "0.2.14" @@ -5125,29 +4063,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "nu-ansi-term" -version = "0.50.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - [[package]] name = "num-bigint" version = "0.4.6" @@ -5194,28 +4109,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-iter" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -5415,43 +4308,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "parquet" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0dbd48ad52d7dccf8ea1b90a3ddbfaea4f69878dd7683e51c507d4bc52b5b27" -dependencies = [ - "ahash", - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-cast 56.2.0", - "arrow-data 56.2.0", - "arrow-ipc 56.2.0", - "arrow-schema 56.2.0", - "arrow-select 56.2.0", - "base64", - "brotli", - "bytes", - "chrono", - "flate2", - "futures", - "half", - "hashbrown 0.16.1", - "lz4_flex 0.11.5", - "num", - "num-bigint", - "object_store", - "paste", - "ring", - "seq-macro", - "simdutf8", - "snap", - "thrift", - "tokio", - "twox-hash", - "zstd", -] - [[package]] name = "parquet" version = "57.1.0" @@ -5459,13 +4315,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be3e4f6d320dd92bfa7d612e265d7d08bba0a240bab86af3425e1d255a511d89" dependencies = [ "ahash", - "arrow-array 57.1.0", - "arrow-buffer 57.1.0", - "arrow-cast 57.1.0", - "arrow-data 57.1.0", - "arrow-ipc 57.1.0", - "arrow-schema 57.1.0", - "arrow-select 57.1.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", "base64", "brotli", "bytes", @@ -5474,7 +4330,7 @@ dependencies = [ "futures", "half", "hashbrown 0.16.1", - "lz4_flex 0.12.0", + "lz4_flex", "num-bigint", "num-integer", "num-traits", @@ -5555,16 +4411,6 @@ dependencies = [ "sha2", ] -[[package]] -name = "petgraph" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" -dependencies = [ - "fixedbitset 0.5.7", - "indexmap", -] - [[package]] name = "petgraph" version = "0.8.3" @@ -5792,16 +4638,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "prost" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" -dependencies = [ - "bytes", - "prost-derive 0.13.5", -] - [[package]] name = "prost" version = "0.14.1" @@ -5809,40 +4645,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" dependencies = [ "bytes", - "prost-derive 0.14.1", -] - -[[package]] -name = "prost-build" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" -dependencies = [ - "heck 0.5.0", - "itertools 0.14.0", - "log", - "multimap", - "once_cell", - "petgraph 0.7.1", - "prettyplease", - "prost 0.13.5", - "prost-types 0.13.5", - "regex", - "syn 2.0.110", - "tempfile", -] - -[[package]] -name = "prost-derive" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" -dependencies = [ - "anyhow", - "itertools 0.14.0", - "proc-macro2", - "quote", - "syn 2.0.110", + "prost-derive", ] [[package]] @@ -5858,22 +4661,13 @@ dependencies = [ "syn 2.0.110", ] -[[package]] -name = "prost-types" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" -dependencies = [ - "prost 0.13.5", -] - [[package]] name = "prost-types" version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" dependencies = [ - "prost 0.14.1", + "prost", ] [[package]] @@ -6732,17 +5526,6 @@ dependencies = [ "windows-sys 0.60.2", ] -[[package]] -name = "sqlparser" -version = "0.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec4b661c54b1e4b603b37873a18c59920e4c51ea8ea2cf527d925424dbd4437c" -dependencies = [ - "log", - "recursive", - "sqlparser_derive", -] - [[package]] name = "sqlparser" version = "0.59.0" @@ -7093,7 +5876,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" dependencies = [ "deranged", - "itoa", "libc", "num-conv", "num_threads", @@ -7265,35 +6047,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "tonic" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" -dependencies = [ - "async-trait", - "axum 0.8.7", - "base64", - "bytes", - "h2 0.4.12", - "http 1.3.1", - "http-body 1.0.1", - "http-body-util", - "hyper 1.8.1", - "hyper-timeout", - "hyper-util", - "percent-encoding", - "pin-project", - "prost 0.13.5", - "socket2 0.5.10", - "tokio", - "tokio-stream", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "tonic" version = "0.14.2" @@ -7323,20 +6076,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tonic-build" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac6f67be712d12f0b41328db3137e0d0757645d8904b4cb7d51cd9c2279e847" -dependencies = [ - "prettyplease", - "proc-macro2", - "prost-build", - "prost-types 0.13.5", - "quote", - "syn 2.0.110", -] - [[package]] name = "tonic-prost" version = "0.14.2" @@ -7344,8 +6083,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" dependencies = [ "bytes", - "prost 0.14.1", - "tonic 0.14.2", + "prost", + "tonic", ] [[package]] @@ -7407,7 +6146,7 @@ name = "tpchgen-arrow" version = "2.0.1" source = "git+https://github.com/clflushopt/tpchgen-rs?rev=e83365a5a9101906eb9f78c5607b83bc59849acf#e83365a5a9101906eb9f78c5607b83bc59849acf" dependencies = [ - "arrow 57.1.0", + "arrow", "tpchgen", ] @@ -7423,18 +6162,6 @@ dependencies = [ "tracing-core", ] -[[package]] -name = "tracing-appender" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf" -dependencies = [ - "crossbeam-channel", - "thiserror 2.0.17", - "time", - "tracing-subscriber", -] - [[package]] name = "tracing-attributes" version = "0.1.31" @@ -7466,33 +6193,15 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - [[package]] name = "tracing-subscriber" version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" dependencies = [ - "matchers", - "nu-ansi-term", - "once_cell", - "regex-automata", "sharded-slab", - "smallvec", "thread_local", - "tracing", "tracing-core", - "tracing-log", ] [[package]] diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index e9efe685..4c96293c 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -8,10 +8,6 @@ default-run = "dfbench" datafusion = { workspace = true } datafusion-proto = { workspace = true } datafusion-distributed = { path = "..", features = ["integration"] } -ballista = { version = "50" } -ballista-executor = { version = "50" } -ballista-scheduler = { version = "50" } -ballista-core = "50" tokio = { version = "1.46.1", features = ["full"] } parquet = { version = "57.1.0" } structopt = { version = "0.3.26" } @@ -41,15 +37,3 @@ path = "src/main.rs" [[bin]] name = "worker" path = "cdk/bin/worker.rs" - -[[bin]] -name = "ballista-http" -path = "cdk/bin/ballista_http.rs" - -[[bin]] -name = "ballista-executor" -path = "cdk/bin/ballista_executor.rs" - -[[bin]] -name = "ballista-scheduler" -path = "cdk/bin/ballista_scheduler.rs" diff --git a/benchmarks/cdk/bin/ballista-bench.ts b/benchmarks/cdk/bin/ballista-bench.ts deleted file mode 100644 index a2755f37..00000000 --- a/benchmarks/cdk/bin/ballista-bench.ts +++ /dev/null @@ -1,97 +0,0 @@ -import path from "path"; -import {Command} from "commander"; -import {z} from 'zod'; -import {BenchmarkRunner, ROOT, runBenchmark, TableSpec} from "./@bench-common"; - -// Remember to port-forward the ballista HTTP server with -// aws ssm start-session --target {host-id} --document-name AWS-StartPortForwardingSession --parameters "portNumber=9002,localPortNumber=9002" - -async function main() { - const program = new Command(); - - program - .option('--dataset ', 'Dataset to run queries on') - .option('-i, --iterations ', 'Number of iterations', '3') - .option('--queries ', 'Specific queries to run', undefined) - .parse(process.argv); - - const options = program.opts(); - - const dataset: string = options.dataset - const iterations = parseInt(options.iterations); - const queries = options.queries?.split(",") ?? [] - - const runner = new BallistaRunner({}); - - const datasetPath = path.join(ROOT, "benchmarks", "data", dataset); - const outputPath = path.join(datasetPath, "remote-results.json") - - await runBenchmark(runner, { - dataset, - iterations, - queries, - outputPath, - }); -} - -const QueryResponse = z.object({ - count: z.number(), - plan: z.string() -}) -type QueryResponse = z.infer - -class BallistaRunner implements BenchmarkRunner { - private url = 'http://localhost:9002'; - - constructor(private readonly options: {}) { - } - - async executeQuery(sql: string): Promise<{ rowCount: number }> { - let response - if (sql.includes("create view")) { - // This is query 15 - let [createView, query, dropView] = sql.split(";") - await this.query(createView); - response = await this.query(query) - await this.query(dropView); - } else { - response = await this.query(sql) - } - - return { rowCount: response.count }; - } - - private async query(sql: string): Promise { - const url = new URL(this.url); - url.searchParams.set('sql', sql); - - const response = await fetch(url.toString()); - - if (!response.ok) { - const msg = await response.text(); - throw new Error(`Query failed: ${response.status} ${msg}`); - } - - const unparsed = await response.json(); - return QueryResponse.parse(unparsed); - } - - async createTables(tables: TableSpec[]): Promise { - let stmt = ''; - for (const table of tables) { - // language=SQL format=false - stmt += ` - DROP TABLE IF EXISTS ${table.name}; - CREATE EXTERNAL TABLE IF NOT EXISTS ${table.name} STORED AS PARQUET LOCATION '${table.s3Path}'; - `; - } - await this.query(stmt); - } - -} - -main() - .catch(err => { - console.error(err) - process.exit(1) - }) diff --git a/benchmarks/cdk/bin/ballista_executor.rs b/benchmarks/cdk/bin/ballista_executor.rs deleted file mode 100644 index 87f660c0..00000000 --- a/benchmarks/cdk/bin/ballista_executor.rs +++ /dev/null @@ -1,36 +0,0 @@ -use ballista::datafusion::execution::runtime_env::RuntimeEnv; -use ballista::datafusion::prelude::SessionConfig; -use ballista_executor::config::Config; -use ballista_executor::executor_process::{ExecutorProcessConfig, start_executor_process}; -use clap::Parser; -use object_store::aws::AmazonS3Builder; -use std::env; -use std::sync::Arc; -use url::Url; - -#[tokio::main] -async fn main() -> Result<(), Box> { - let opt = Config::parse(); - - let mut config: ExecutorProcessConfig = opt.try_into()?; - - let bucket = env::var("BUCKET").unwrap_or("datafusion-distributed-benchmarks".to_string()); - let s3_url = Url::parse(&format!("s3://{bucket}"))?; - - let s3 = Arc::new( - AmazonS3Builder::from_env() - .with_bucket_name(s3_url.host().unwrap().to_string()) - .build()?, - ); - let runtime_env = Arc::new(RuntimeEnv::default()); - runtime_env.register_object_store(&s3_url, s3); - - config.override_runtime_producer = Some(Arc::new( - move |_: &SessionConfig| -> ballista::datafusion::common::Result> { - Ok(runtime_env.clone()) - }, - )); - - start_executor_process(Arc::new(config)).await?; - Ok(()) -} diff --git a/benchmarks/cdk/bin/ballista_http.rs b/benchmarks/cdk/bin/ballista_http.rs deleted file mode 100644 index 948aa1e1..00000000 --- a/benchmarks/cdk/bin/ballista_http.rs +++ /dev/null @@ -1,124 +0,0 @@ -use axum::{Json, Router, extract::Query, http::StatusCode, routing::get}; -use ballista::datafusion::common::instant::Instant; -use ballista::datafusion::execution::SessionStateBuilder; -use ballista::datafusion::execution::runtime_env::RuntimeEnv; -use ballista::datafusion::physical_plan::displayable; -use ballista::datafusion::physical_plan::execute_stream; -use ballista::datafusion::prelude::SessionConfig; -use ballista::datafusion::prelude::SessionContext; -use ballista::prelude::*; -use futures::{StreamExt, TryFutureExt}; -use log::{error, info}; -use object_store::aws::AmazonS3Builder; -use serde::Serialize; -use std::collections::HashMap; -use std::error::Error; -use std::fmt::Display; -use std::sync::Arc; -use structopt::StructOpt; -use url::Url; - -#[derive(Serialize)] -struct QueryResult { - plan: String, - count: usize, -} - -#[derive(Debug, StructOpt, Clone)] -#[structopt(about = "worker spawn command")] -struct Cmd { - /// The bucket name. - #[structopt(long, default_value = "datafusion-distributed-benchmarks")] - bucket: String, -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - env_logger::builder() - .filter_level(log::LevelFilter::Info) - .parse_default_env() - .init(); - - let cmd = Cmd::from_args(); - - const LISTENER_ADDR: &str = "0.0.0.0:9002"; - - info!("Starting HTTP listener on {LISTENER_ADDR}..."); - let listener = tokio::net::TcpListener::bind(LISTENER_ADDR).await?; - - // Register S3 object store - let s3_url = Url::parse(&format!("s3://{}", cmd.bucket))?; - - info!("Building shared SessionContext for the whole lifetime of the HTTP listener..."); - let s3 = Arc::new( - AmazonS3Builder::from_env() - .with_bucket_name(s3_url.host().unwrap().to_string()) - .build()?, - ); - let runtime_env = Arc::new(RuntimeEnv::default()); - runtime_env.register_object_store(&s3_url, s3); - - let config = SessionConfig::new_with_ballista().with_ballista_job_name("Benchmarks"); - - let state = SessionStateBuilder::new() - .with_config(config) - .with_default_features() - .with_runtime_env(Arc::clone(&runtime_env)) - .build(); - let ctx = SessionContext::remote_with_state("df://localhost:50050", state).await?; - - let http_server = axum::serve( - listener, - Router::new().route( - "/", - get(move |Query(params): Query>| { - let ctx = ctx.clone(); - - async move { - let sql = params.get("sql").ok_or(err("Missing 'sql' parameter"))?; - - let mut df_opt = None; - for sql in sql.split(";") { - if sql.trim().is_empty() { - continue; - } - let df = ctx.sql(sql).await.map_err(err)?; - df_opt = Some(df); - } - let Some(df) = df_opt else { - return Err(err("Empty 'sql' parameter")); - }; - - let start = Instant::now(); - - info!("Executing query..."); - let physical = df.create_physical_plan().await.map_err(err)?; - let mut stream = - execute_stream(physical.clone(), ctx.task_ctx()).map_err(err)?; - let mut count = 0; - while let Some(batch) = stream.next().await { - count += batch.map_err(err)?.num_rows(); - info!("Gathered {count} rows, query still in progress..") - } - let plan = displayable(physical.as_ref()).indent(true).to_string(); - let elapsed = start.elapsed(); - let ms = elapsed.as_secs_f64() * 1000.0; - info!("Returned {count} rows in {ms} ms"); - - Ok::<_, (StatusCode, String)>(Json(QueryResult { count, plan })) - } - .inspect_err(|(_, msg)| { - error!("Error executing query: {msg}"); - }) - }), - ), - ); - - info!("Started listener HTTP server in {LISTENER_ADDR}"); - http_server.await?; - Ok(()) -} - -fn err(s: impl Display) -> (StatusCode, String) { - (StatusCode::INTERNAL_SERVER_ERROR, s.to_string()) -} diff --git a/benchmarks/cdk/bin/ballista_scheduler.rs b/benchmarks/cdk/bin/ballista_scheduler.rs deleted file mode 100644 index ba0b1f2d..00000000 --- a/benchmarks/cdk/bin/ballista_scheduler.rs +++ /dev/null @@ -1,62 +0,0 @@ -use ballista::datafusion::execution::runtime_env::RuntimeEnv; -use ballista::datafusion::execution::{SessionState, SessionStateBuilder}; -use ballista::datafusion::prelude::SessionConfig; -use ballista_core::error::BallistaError; -use ballista_core::extension::SessionConfigExt; -use ballista_scheduler::cluster::BallistaCluster; -use ballista_scheduler::config::{Config, SchedulerConfig}; -use ballista_scheduler::scheduler_process::start_server; -use clap::Parser; -use object_store::aws::AmazonS3Builder; -use std::env; -use std::sync::Arc; -use url::Url; - -fn main() -> Result<(), Box> { - let runtime = tokio::runtime::Builder::new_multi_thread() - .enable_io() - .enable_time() - .thread_stack_size(32 * 1024 * 1024) // 32MB - .build()?; - - runtime.block_on(inner()) -} - -async fn inner() -> Result<(), Box> { - let opt = Config::parse(); - - let addr = format!("{}:{}", opt.bind_host, opt.bind_port); - let addr = addr - .parse() - .map_err(|e: std::net::AddrParseError| BallistaError::Configuration(e.to_string()))?; - - let bucket = env::var("BUCKET").unwrap_or("datafusion-distributed-benchmarks".to_string()); - let s3_url = Url::parse(&format!("s3://{bucket}"))?; - - let s3 = Arc::new( - AmazonS3Builder::from_env() - .with_bucket_name(s3_url.host().unwrap().to_string()) - .build()?, - ); - let runtime_env = Arc::new(RuntimeEnv::default()); - runtime_env.register_object_store(&s3_url, s3); - - let config: SchedulerConfig = opt.try_into()?; - let config = config.with_override_config_producer(Arc::new(|| { - SessionConfig::new_with_ballista().with_information_schema(true) - })); - let config = config.with_override_session_builder(Arc::new( - move |cfg: SessionConfig| -> ballista::datafusion::common::Result { - Ok(SessionStateBuilder::new() - .with_config(cfg) - .with_runtime_env(runtime_env.clone()) - .with_default_features() - .build()) - }, - )); - - let cluster = BallistaCluster::new_from_config(&config).await?; - start_server(cluster, addr, Arc::new(config)).await?; - - Ok(()) -} diff --git a/benchmarks/cdk/bin/cdk.ts b/benchmarks/cdk/bin/cdk.ts index 442fe003..d209dc1e 100644 --- a/benchmarks/cdk/bin/cdk.ts +++ b/benchmarks/cdk/bin/cdk.ts @@ -3,7 +3,6 @@ import * as cdk from 'aws-cdk-lib/core'; import {CdkStack} from '../lib/cdk-stack'; import {DATAFUSION_DISTRIBUTED_ENGINE} from "../lib/datafusion-distributed"; import {TRINO_ENGINE} from "../lib/trino"; -import {BALLISTA_ENGINE} from "../lib/ballista"; const app = new cdk.App(); @@ -13,7 +12,6 @@ const config = { engines: [ DATAFUSION_DISTRIBUTED_ENGINE, TRINO_ENGINE, - BALLISTA_ENGINE ] }; diff --git a/benchmarks/cdk/lib/ballista.ts b/benchmarks/cdk/lib/ballista.ts deleted file mode 100644 index 5e138ee3..00000000 --- a/benchmarks/cdk/lib/ballista.ts +++ /dev/null @@ -1,212 +0,0 @@ -import { - AfterEc2MachinesContext, - BeforeEc2MachinesContext, - QueryEngine, - ROOT, - sendCommandsUnconditionally, - OnEc2MachinesContext -} from "./cdk-stack"; -import * as s3assets from "aws-cdk-lib/aws-s3-assets"; -import path from "path"; -import {execSync} from "child_process"; - -let ballistaServerBinary: s3assets.Asset -let ballistaSchedulerBinary: s3assets.Asset -let ballistaExecutorBinary: s3assets.Asset - -export const BALLISTA_ENGINE: QueryEngine = { - beforeEc2Machines(ctx: BeforeEc2MachinesContext): void { - console.log('Building Ballista server binary...'); - execSync('cargo zigbuild -p datafusion-distributed-benchmarks --release --bin ballista-http --target x86_64-unknown-linux-gnu', { - cwd: ROOT, - stdio: 'inherit', - }); - console.log('Ballista server binary built successfully'); - - console.log('Building Ballista scheduler...'); - execSync('cargo zigbuild -p datafusion-distributed-benchmarks --release --bin ballista-scheduler --target x86_64-unknown-linux-gnu', { - cwd: ROOT, - stdio: 'inherit', - }); - console.log('Ballista scheduler built successfully'); - - console.log('Building Ballista executor...'); - execSync('cargo zigbuild -p datafusion-distributed-benchmarks --release --bin ballista-executor --target x86_64-unknown-linux-gnu', { - cwd: ROOT, - stdio: 'inherit', - }); - console.log('Ballista scheduler built successfully'); - - ballistaServerBinary = new s3assets.Asset(ctx.scope, 'BallistaServerBinary', { - path: path.join(ROOT, 'target/x86_64-unknown-linux-gnu/release/ballista-http'), - }) - - ballistaSchedulerBinary = new s3assets.Asset(ctx.scope, 'BallistaSchedulerBinary', { - path: path.join(ROOT, 'target/x86_64-unknown-linux-gnu/release/ballista-scheduler'), - }) - - ballistaExecutorBinary = new s3assets.Asset(ctx.scope, 'BallistaExecutorBinary', { - path: path.join(ROOT, 'target/x86_64-unknown-linux-gnu/release/ballista-executor'), - }) - - ballistaServerBinary.grantRead(ctx.role) - ballistaSchedulerBinary.grantRead(ctx.role) - ballistaExecutorBinary.grantRead(ctx.role) - }, - onEc2Machine(ctx: OnEc2MachinesContext): void { - const isScheduler = ctx.instanceIdx === 0; - ctx.instanceUserData.addCommands( - // Download pre-compiled Ballista binaries from S3 - `aws s3 cp s3://${ballistaSchedulerBinary.s3BucketName}/${ballistaSchedulerBinary.s3ObjectKey} /usr/local/bin/ballista-scheduler`, - 'chmod +x /usr/local/bin/ballista-scheduler', - `aws s3 cp s3://${ballistaExecutorBinary.s3BucketName}/${ballistaExecutorBinary.s3ObjectKey} /usr/local/bin/ballista-executor`, - 'chmod +x /usr/local/bin/ballista-executor', - `aws s3 cp s3://${ballistaServerBinary.s3BucketName}/${ballistaServerBinary.s3ObjectKey} /usr/local/bin/ballista-http`, - 'chmod +x /usr/local/bin/ballista-http', - - // Create Ballista directories - 'mkdir -p /var/ballista/scheduler', - 'mkdir -p /var/ballista/executor', - 'mkdir -p /var/ballista/logs', - - // Create Ballista scheduler systemd service (coordinator only) - ...(isScheduler ? [ - `cat > /etc/systemd/system/ballista-scheduler.service << 'BALLISTA_EOF' -[Unit] -Description=Ballista Scheduler -After=network.target - -[Service] -Type=simple -ExecStart=/usr/local/bin/ballista-scheduler \\ - --bind-host 0.0.0.0 \\ - --bind-port 50050 -Restart=on-failure -RestartSec=5 -User=root -WorkingDirectory=/var/ballista/scheduler -StandardOutput=append:/var/ballista/logs/scheduler.log -StandardError=append:/var/ballista/logs/scheduler.log - -[Install] -WantedBy=multi-user.target -BALLISTA_EOF` - ] : []), - - // Create Ballista executor systemd service (all nodes, will be reconfigured for workers) - `cat > /etc/systemd/system/ballista-executor.service << 'BALLISTA_EOF' -[Unit] -Description=Ballista Executor -After=network.target${isScheduler ? ' ballista-scheduler.service' : ''} -${isScheduler ? 'Requires=ballista-scheduler.service' : ''} - -[Service] -Type=simple -ExecStart=/usr/local/bin/ballista-executor \\ - --bind-host 0.0.0.0 \\ - --bind-port 50051 \\ - --work-dir /var/ballista/executor \\ - --scheduler-host localhost \\ - --scheduler-port 50050 -Restart=on-failure -RestartSec=5 -User=root -Environment="BUCKET=${ctx.bucketName}" -WorkingDirectory=/var/ballista/executor -StandardOutput=append:/var/ballista/logs/executor.log -StandardError=append:/var/ballista/logs/executor.log - -[Install] -WantedBy=multi-user.target -BALLISTA_EOF`, - - // Create HTTP server systemd service (coordinator only for now) - ...(isScheduler ? [ - `aws s3 cp s3://${ballistaServerBinary.s3BucketName}/${ballistaServerBinary.s3ObjectKey} /usr/local/bin/ballista-http`, - 'chmod +x /usr/local/bin/ballista-http', - `cat > /etc/systemd/system/ballista-http.service << 'BALLISTA_EOF' -[Unit] -Description=Ballista HTTP Server -After=network.target ballista-scheduler.service -Requires=ballista-scheduler.service - -[Service] -Type=simple -ExecStart=/usr/local/bin/ballista-http --bucket ${ctx.bucketName} -Restart=on-failure -RestartSec=5 -User=root -Environment="RUST_LOG=info" -WorkingDirectory=/var/ballista -StandardOutput=append:/var/ballista/logs/http.log -StandardError=append:/var/ballista/logs/http.log - -[Install] -WantedBy=multi-user.target -BALLISTA_EOF` - ] : []), - - // Reload systemd and enable services - 'systemctl daemon-reload', - - // Enable and start scheduler (coordinator only) - ...(isScheduler ? [ - 'systemctl enable ballista-scheduler', - 'systemctl start ballista-scheduler', - // Wait for scheduler to be ready - 'sleep 5' - ] : []), - - // Enable and start executor (all nodes) - 'systemctl enable ballista-executor', - 'systemctl start ballista-executor', - - // Enable and start HTTP server (coordinator only) - ...(isScheduler ? [ - 'systemctl enable ballista-http', - 'systemctl start ballista-http' - ] : []) - ) - - }, - afterEc2Machines(ctx: AfterEc2MachinesContext) { - const [scheduler, ...executors] = ctx.instances - - // Reconfigure executors on worker nodes to point to scheduler. The executor in the machine holding the scheduler - // communicates to it using localhost, so no need to update it with scheduler.instancePrivateIp. - sendCommandsUnconditionally( - ctx.scope, - "ConfigureBallistaExecutors", - [scheduler, ...executors], - [ - `cat > /etc/systemd/system/ballista-executor.service << 'BALLISTA_EOF' -[Unit] -Description=Ballista Executor -After=network.target - -[Service] -Type=simple -ExecStart=/usr/local/bin/ballista-executor \\ - --bind-host 0.0.0.0 \\ - --bind-port 50051 \\ - --work-dir /var/ballista/executor \\ - --scheduler-host ${scheduler.instancePrivateIp} \\ - --scheduler-port 50050 -Restart=on-failure -RestartSec=5 -User=root -Environment="BUCKET=${ctx.bucketName}" -WorkingDirectory=/var/ballista/executor -StandardOutput=append:/var/ballista/logs/executor.log -StandardError=append:/var/ballista/logs/executor.log - -[Install] -WantedBy=multi-user.target -BALLISTA_EOF`, - 'systemctl daemon-reload', - 'systemctl restart ballista-executor', - ] - ) - } -} - diff --git a/benchmarks/cdk/lib/cdk-stack.ts b/benchmarks/cdk/lib/cdk-stack.ts index f20ad86f..0709c9eb 100644 --- a/benchmarks/cdk/lib/cdk-stack.ts +++ b/benchmarks/cdk/lib/cdk-stack.ts @@ -4,7 +4,6 @@ import * as s3 from 'aws-cdk-lib/aws-s3'; import * as iam from 'aws-cdk-lib/aws-iam'; import {Construct} from 'constructs'; import {DATAFUSION_DISTRIBUTED_ENGINE} from "./datafusion-distributed"; -import {BALLISTA_ENGINE} from "./ballista"; import {TRINO_ENGINE} from "./trino"; import {SPARK_ENGINE} from "./spark"; import path from "path"; @@ -17,7 +16,6 @@ if (USER_DATA_CAUSES_REPLACEMENT) { const ENGINES = [ DATAFUSION_DISTRIBUTED_ENGINE, - BALLISTA_ENGINE, TRINO_ENGINE, SPARK_ENGINE ] diff --git a/benchmarks/cdk/package.json b/benchmarks/cdk/package.json index 719d58d1..ed800073 100644 --- a/benchmarks/cdk/package.json +++ b/benchmarks/cdk/package.json @@ -11,7 +11,6 @@ "cdk": "cdk", "sync-bucket": "aws s3 sync ../data s3://datafusion-distributed-benchmarks/", "datafusion-bench": "npx ts-node bin/datafusion-bench.ts", - "ballista-bench": "npx ts-node bin/ballista-bench.ts", "trino-bench": "npx ts-node bin/trino-bench.ts", "spark-bench": "npx ts-node bin/spark-bench.ts" }, From 92e5b0ba2fbf3b636b2b0f1dd26633a30f9a44b6 Mon Sep 17 00:00:00 2001 From: Gabriel Musat Mestre Date: Wed, 14 Jan 2026 09:37:42 +0100 Subject: [PATCH 21/27] Adapt codebase for DataDog --- Cargo.lock | 410 ++++++++++++++++++------------------------ Cargo.toml | 29 ++- benchmarks/Cargo.toml | 4 +- cli/Cargo.toml | 2 +- 4 files changed, 204 insertions(+), 241 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 204b8497..11670c91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,15 +148,16 @@ checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] name = "apache-avro" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a033b4ced7c585199fb78ef50fca7fe2f444369ec48080c5fd072efa1a03cc7" +checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" dependencies = [ "bigdecimal", "bon", - "bzip2 0.6.1", + "bzip2", "crc32fast", "digest", + "liblzma", "log", "miniz_oxide", "num-bigint", @@ -171,7 +172,6 @@ dependencies = [ "strum_macros 0.27.2", "thiserror 2.0.17", "uuid", - "xz2", "zstd", ] @@ -208,8 +208,7 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb372a7cbcac02a35d3fb7b3fc1f969ec078e871f9bb899bf00a2e1809bec8a3" +source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" dependencies = [ "arrow-arith", "arrow-array", @@ -229,8 +228,7 @@ dependencies = [ [[package]] name = "arrow-arith" version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f377dcd19e440174596d83deb49cd724886d91060c07fec4f67014ef9d54049" +source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" dependencies = [ "arrow-array", "arrow-buffer", @@ -243,8 +241,7 @@ dependencies = [ [[package]] name = "arrow-array" version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eaff85a44e9fa914660fb0d0bb00b79c4a3d888b5334adb3ea4330c84f002" +source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" dependencies = [ "ahash", "arrow-buffer", @@ -262,8 +259,7 @@ dependencies = [ [[package]] name = "arrow-buffer" version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2819d893750cb3380ab31ebdc8c68874dd4429f90fd09180f3c93538bd21626" +source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" dependencies = [ "bytes", "half", @@ -274,8 +270,7 @@ dependencies = [ [[package]] name = "arrow-cast" version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d131abb183f80c450d4591dc784f8d7750c50c6e2bc3fcaad148afc8361271" +source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" dependencies = [ "arrow-array", "arrow-buffer", @@ -296,8 +291,7 @@ dependencies = [ [[package]] name = "arrow-csv" version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2275877a0e5e7e7c76954669366c2aa1a829e340ab1f612e647507860906fb6b" +source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" dependencies = [ "arrow-array", "arrow-cast", @@ -311,8 +305,7 @@ dependencies = [ [[package]] name = "arrow-data" version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05738f3d42cb922b9096f7786f606fcb8669260c2640df8490533bb2fa38c9d3" +source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" dependencies = [ "arrow-buffer", "arrow-schema", @@ -324,8 +317,7 @@ dependencies = [ [[package]] name = "arrow-flight" version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b5f57c3d39d1b1b7c1376a772ea86a131e7da310aed54ebea9363124bb885e3" +source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" dependencies = [ "arrow-array", "arrow-buffer", @@ -338,14 +330,12 @@ dependencies = [ "prost", "prost-types", "tonic", - "tonic-prost", ] [[package]] name = "arrow-ipc" version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d09446e8076c4b3f235603d9ea7c5494e73d441b01cd61fb33d7254c11964b3" +source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" dependencies = [ "arrow-array", "arrow-buffer", @@ -360,8 +350,7 @@ dependencies = [ [[package]] name = "arrow-json" version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "371ffd66fa77f71d7628c63f209c9ca5341081051aa32f9c8020feb0def787c0" +source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" dependencies = [ "arrow-array", "arrow-buffer", @@ -384,8 +373,7 @@ dependencies = [ [[package]] name = "arrow-ord" version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc94fc7adec5d1ba9e8cd1b1e8d6f72423b33fe978bf1f46d970fafab787521" +source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" dependencies = [ "arrow-array", "arrow-buffer", @@ -397,8 +385,7 @@ dependencies = [ [[package]] name = "arrow-row" version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "169676f317157dc079cc5def6354d16db63d8861d61046d2f3883268ced6f99f" +source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" dependencies = [ "arrow-array", "arrow-buffer", @@ -410,8 +397,7 @@ dependencies = [ [[package]] name = "arrow-schema" version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27609cd7dd45f006abae27995c2729ef6f4b9361cde1ddd019dc31a5aa017e0" +source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" dependencies = [ "serde_core", "serde_json", @@ -420,8 +406,7 @@ dependencies = [ [[package]] name = "arrow-select" version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae980d021879ea119dd6e2a13912d81e64abed372d53163e804dfe84639d8010" +source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" dependencies = [ "ahash", "arrow-array", @@ -434,8 +419,7 @@ dependencies = [ [[package]] name = "arrow-string" version = "57.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf35e8ef49dcf0c5f6d175edee6b8af7b45611805333129c541a8b89a0fc0534" +source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" dependencies = [ "arrow-array", "arrow-buffer", @@ -450,19 +434,14 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.19" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" +checksum = "d10e4f991a553474232bc0a31799f6d24b034a84c0971d80d2e2f78b2e576e40" dependencies = [ - "bzip2 0.5.2", - "flate2", - "futures-core", - "memchr", + "compression-codecs", + "compression-core", "pin-project-lite", "tokio", - "xz2", - "zstd", - "zstd-safe", ] [[package]] @@ -1208,15 +1187,6 @@ dependencies = [ "either", ] -[[package]] -name = "bzip2" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" -dependencies = [ - "bzip2-sys", -] - [[package]] name = "bzip2" version = "0.6.1" @@ -1226,16 +1196,6 @@ dependencies = [ "libbz2-rs-sys", ] -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "castaway" version = "0.2.4" @@ -1434,6 +1394,27 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "compression-codecs" +version = "0.4.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00828ba6fd27b45a448e57dbfe84f1029d4c9f26b368157e9a448a5f49a2ec2a" +dependencies = [ + "bzip2", + "compression-core", + "flate2", + "liblzma", + "memchr", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" + [[package]] name = "console" version = "0.1.0" @@ -1525,21 +1506,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - [[package]] name = "crc32fast" version = "1.5.0" @@ -1733,18 +1699,17 @@ dependencies = [ [[package]] name = "datafusion" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ba7cb113e9c0bedf9e9765926031e132fa05a1b09ba6e93a6d1a4d7044457b8" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", "arrow-schema", "async-trait", "bytes", - "bzip2 0.6.1", + "bzip2", "chrono", "datafusion-catalog", "datafusion-catalog-listing", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-common-runtime", "datafusion-datasource", "datafusion-datasource-arrow", @@ -1771,6 +1736,7 @@ dependencies = [ "flate2", "futures", "itertools 0.14.0", + "liblzma", "log", "object_store", "parking_lot", @@ -1783,20 +1749,18 @@ dependencies = [ "tokio", "url", "uuid", - "xz2", "zstd", ] [[package]] name = "datafusion-catalog" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a3a799f914a59b1ea343906a0486f17061f39509af74e874a866428951130d" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-common-runtime", "datafusion-datasource", "datafusion-execution", @@ -1815,13 +1779,12 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db1b113c80d7a0febcd901476a57aef378e717c54517a163ed51417d87621b0" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", "async-trait", "datafusion-catalog", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-datasource", "datafusion-execution", "datafusion-expr", @@ -1849,7 +1812,7 @@ dependencies = [ "chrono", "clap 4.5.53", "datafusion", - "datafusion-common", + "datafusion-common 51.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "dirs", "env_logger", "futures", @@ -1869,6 +1832,25 @@ name = "datafusion-common" version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c10f7659e96127d25e8366be7c8be4109595d6a2c3eac70421f380a7006a1b0" +dependencies = [ + "ahash", + "arrow", + "arrow-ipc", + "chrono", + "half", + "hashbrown 0.14.5", + "indexmap", + "libc", + "log", + "paste", + "tokio", + "web-time", +] + +[[package]] +name = "datafusion-common" +version = "51.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "ahash", "apache-avro", @@ -1893,8 +1875,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b92065bbc6532c6651e2f7dd30b55cba0c7a14f860c7e1d15f165c41a1868d95" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "futures", "log", @@ -1904,16 +1885,15 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde13794244bc7581cd82f6fff217068ed79cdc344cafe4ab2c3a1c3510b38d6" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", "async-compression", "async-trait", "bytes", - "bzip2 0.6.1", + "bzip2", "chrono", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", @@ -1926,27 +1906,26 @@ dependencies = [ "futures", "glob", "itertools 0.14.0", + "liblzma", "log", "object_store", "rand 0.9.2", "tokio", "tokio-util", "url", - "xz2", "zstd", ] [[package]] name = "datafusion-datasource-arrow" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804fa9b4ecf3157982021770617200ef7c1b2979d57bec9044748314775a9aea" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", "arrow-ipc", "async-trait", "bytes", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-common-runtime", "datafusion-datasource", "datafusion-execution", @@ -1963,14 +1942,13 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388ed8be535f562cc655b9c3d22edbfb0f1a50a25c242647a98b6d92a75b55a1" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "apache-avro", "arrow", "async-trait", "bytes", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-datasource", "datafusion-physical-expr-common", "datafusion-physical-plan", @@ -1983,13 +1961,12 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a1641a40b259bab38131c5e6f48fac0717bedb7dc93690e604142a849e0568" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-common-runtime", "datafusion-datasource", "datafusion-execution", @@ -2006,13 +1983,12 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adeacdb00c1d37271176f8fb6a1d8ce096baba16ea7a4b2671840c5c9c64fe85" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-common-runtime", "datafusion-datasource", "datafusion-execution", @@ -2028,13 +2004,12 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d0b60ffd66f28bfb026565d62b0a6cbc416da09814766a3797bba7d85a3cd9" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-common-runtime", "datafusion-datasource", "datafusion-execution", @@ -2147,19 +2122,17 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b99e13947667b36ad713549237362afb054b2d8f8cc447751e23ec61202db07" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" [[package]] name = "datafusion-execution" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63695643190679037bc946ad46a263b62016931547bf119859c511f7ff2f5178" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-expr", "futures", "log", @@ -2174,13 +2147,12 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a4787cbf5feb1ab351f789063398f67654a6df75c4d37d7f637dc96f951a91" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", "async-trait", "chrono", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-doc", "datafusion-expr-common", "datafusion-functions-aggregate-common", @@ -2197,11 +2169,10 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce2fb1b8c15c9ac45b0863c30b268c69dc9ee7a1ee13ecf5d067738338173dc" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "indexmap", "itertools 0.14.0", "paste", @@ -2210,8 +2181,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "794a9db7f7b96b3346fc007ff25e994f09b8f0511b4cf7dff651fadfe3ebb28f" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", "arrow-buffer", @@ -2219,7 +2189,7 @@ dependencies = [ "blake2", "blake3", "chrono", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-doc", "datafusion-execution", "datafusion-expr", @@ -2240,12 +2210,11 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c25210520a9dcf9c2b2cbbce31ebd4131ef5af7fc60ee92b266dc7d159cb305" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "ahash", "arrow", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-doc", "datafusion-execution", "datafusion-expr", @@ -2261,12 +2230,11 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f4a66f3b87300bb70f4124b55434d2ae3fe80455f3574701d0348da040b55d" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "ahash", "arrow", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-expr-common", "datafusion-physical-expr-common", ] @@ -2274,12 +2242,11 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae5c06eed03918dc7fe7a9f082a284050f0e9ecf95d72f57712d1496da03b8c4" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", "arrow-ord", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-doc", "datafusion-execution", "datafusion-expr", @@ -2297,13 +2264,12 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4fed1d71738fbe22e2712d71396db04c25de4111f1ec252b8f4c6d3b25d7f5" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", "async-trait", "datafusion-catalog", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-expr", "datafusion-physical-plan", "parking_lot", @@ -2313,11 +2279,10 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d92206aa5ae21892f1552b4d61758a862a70956e6fd7a95cb85db1de74bc6d1" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-doc", "datafusion-expr", "datafusion-functions-window-common", @@ -2331,18 +2296,16 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53ae9bcc39800820d53a22d758b3b8726ff84a5a3e24cecef04ef4e5fdf1c7cc" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-physical-expr-common", ] [[package]] name = "datafusion-macros" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1063ad4c9e094b3f798acee16d9a47bd7372d9699be2de21b05c3bd3f34ab848" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "datafusion-doc", "quote", @@ -2352,12 +2315,11 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35f9ec5d08b87fd1893a30c2929f2559c2f9806ca072d8fefca5009dc0f06a" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", "chrono", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-expr", "datafusion-expr-common", "datafusion-physical-expr", @@ -2372,12 +2334,11 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c30cc8012e9eedcb48bbe112c6eff4ae5ed19cf3003cb0f505662e88b7014c5d" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "ahash", "arrow", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-expr", "datafusion-expr-common", "datafusion-functions-aggregate-common", @@ -2389,16 +2350,16 @@ dependencies = [ "parking_lot", "paste", "petgraph", + "tokio", ] [[package]] name = "datafusion-physical-expr-adapter" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9ff2dbd476221b1f67337699eff432781c4e6e1713d2aefdaa517dfbf79768" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-expr", "datafusion-functions", "datafusion-physical-expr", @@ -2409,12 +2370,11 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90da43e1ec550b172f34c87ec68161986ced70fd05c8d2a2add66eef9c276f03" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "ahash", "arrow", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-expr-common", "hashbrown 0.14.5", "itertools 0.14.0", @@ -2423,11 +2383,10 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce9804f799acd7daef3be7aaffe77c0033768ed8fdbf5fb82fc4c5f2e6bc14e6" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-execution", "datafusion-expr", "datafusion-expr-common", @@ -2442,8 +2401,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0acf0ad6b6924c6b1aa7d213b181e012e2d3ec0a64ff5b10ee6282ab0f8532ac" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "ahash", "arrow", @@ -2451,7 +2409,7 @@ dependencies = [ "arrow-schema", "async-trait", "chrono", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", @@ -2473,14 +2431,13 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d368093a98a17d1449b1083ac22ed16b7128e4c67789991869480d8c4a40ecb9" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", "chrono", "datafusion-catalog", "datafusion-catalog-listing", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-datasource", "datafusion-datasource-arrow", "datafusion-datasource-csv", @@ -2500,22 +2457,20 @@ dependencies = [ [[package]] name = "datafusion-proto-common" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b6aef3d5e5c1d2bc3114c4876730cb76a9bdc5a8df31ef1b6db48f0c1671895" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "prost", ] [[package]] name = "datafusion-pruning" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac2c2498a1f134a9e11a9f5ed202a2a7d7e9774bd9249295593053ea3be999db" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-datasource", "datafusion-expr-common", "datafusion-physical-expr", @@ -2528,11 +2483,10 @@ dependencies = [ [[package]] name = "datafusion-session" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f96eebd17555386f459037c65ab73aae8df09f464524c709d6a3134ad4f4776" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "async-trait", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-execution", "datafusion-expr", "datafusion-physical-plan", @@ -2542,13 +2496,12 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "51.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fc195fe60634b2c6ccfd131b487de46dc30eccae8a3c35a13f136e7f440414f" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" dependencies = [ "arrow", "bigdecimal", "chrono", - "datafusion-common", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", "datafusion-expr", "indexmap", "log", @@ -3767,6 +3720,26 @@ version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +[[package]] +name = "liblzma" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73c36d08cad03a3fbe2c4e7bb3a9e84c57e4ee4135ed0b065cade3d98480c648" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01b9596486f6d60c3bbe644c0e1be1aa6ccc472ad630fe8927b456973d7cb736" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "libm" version = "0.2.15" @@ -3868,27 +3841,6 @@ dependencies = [ "twox-hash", ] -[[package]] -name = "lzma-rust2" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c60a23ffb90d527e23192f1246b14746e2f7f071cb84476dd879071696c18a4a" -dependencies = [ - "crc", - "sha2", -] - -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "mac_address" version = "1.1.8" @@ -4640,9 +4592,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.14.1" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", "prost-derive", @@ -4650,9 +4602,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.14.1" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", "itertools 0.14.0", @@ -4663,9 +4615,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.14.1" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ "prost", ] @@ -6049,9 +6001,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.14.2" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" +checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" dependencies = [ "async-trait", "axum 0.8.7", @@ -6066,8 +6018,8 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "socket2 0.6.1", - "sync_wrapper", + "prost", + "socket2 0.5.10", "tokio", "tokio-stream", "tower", @@ -6076,17 +6028,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tonic-prost" -version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" -dependencies = [ - "bytes", - "prost", - "tonic", -] - [[package]] name = "tower" version = "0.5.2" @@ -6841,15 +6782,6 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - [[package]] name = "yansi" version = "1.0.1" @@ -6975,13 +6907,13 @@ dependencies = [ [[package]] name = "zip" -version = "6.0.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" +checksum = "caa8cd6af31c3b31c6631b8f483848b91589021b28fffe50adada48d4f4d2ed1" dependencies = [ "aes", "arbitrary", - "bzip2 0.6.1", + "bzip2", "constant_time_eq", "crc32fast", "deflate64", @@ -6989,7 +6921,7 @@ dependencies = [ "getrandom 0.3.4", "hmac", "indexmap", - "lzma-rust2", + "liblzma", "memchr", "pbkdf2", "ppmd-rust", @@ -7045,3 +6977,13 @@ dependencies = [ "cc", "pkg-config", ] + +[[patch.unused]] +name = "arrow-avro" +version = "57.1.0" +source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" + +[[patch.unused]] +name = "datafusion-substrait" +version = "51.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" diff --git a/Cargo.toml b/Cargo.toml index 33c77388..acf203b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ arrow-flight = "57.1.0" arrow-select = "57.1.0" async-trait = "0.1.88" tokio = { version = "1.46.1", features = ["full"] } -tonic = { version = "0.14.1", features = ["transport"] } +tonic = { version = "0.13", features = ["transport"] } tower = "0.5.2" http = "1.3.1" itertools = "0.14.0" @@ -32,7 +32,7 @@ url = "2.5.4" uuid = "1.17.0" delegate = "0.13.4" dashmap = "6.1.0" -prost = "0.14.0" +prost = "0.13" rand = "0.8.5" object_store = "0.12.3" bytes = "1.10.1" @@ -49,7 +49,7 @@ arrow = { version = "57.1.0", optional = true } hyper-util = { version = "0.1.16", optional = true } pretty_assertions = { version = "1.4", optional = true } reqwest = { version = "0.12", optional = true } -zip = { version = "6.0", optional = true } +zip = { version = "4.0", optional = true } [features] avro = ["datafusion/avro"] @@ -80,4 +80,25 @@ tokio-stream = "0.1.17" hyper-util = "0.1.16" pretty_assertions = "1.4" reqwest = "0.12" -zip = "6.0" +zip = "4.0" + +[patch.crates-io] +datafusion = { git = "https://github.com/DataDog/datafusion", rev = "3ae66138c1d6551b29a0039059570c9927a27d52", package = "datafusion" } +datafusion-substrait = { git = "https://github.com/DataDog/datafusion", rev = "3ae66138c1d6551b29a0039059570c9927a27d52", package = "datafusion-substrait" } +datafusion-proto = { git = "https://github.com/DataDog/datafusion", rev = "3ae66138c1d6551b29a0039059570c9927a27d52", package = "datafusion-proto" } +datafusion-proto-common = { git = "https://github.com/DataDog/datafusion", rev = "3ae66138c1d6551b29a0039059570c9927a27d52", package = "datafusion-proto-common" } +arrow = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow" } +arrow-flight = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-flight" } +arrow-select = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-select" } +arrow-ord = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-ord" } +arrow-array = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-array" } +arrow-data = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-data" } +arrow-ipc = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-ipc" } +arrow-schema = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-schema" } +arrow-cast = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-cast" } +arrow-buffer = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-buffer" } +arrow-row = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-row" } +arrow-string = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-string" } +arrow-json = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-json" } +arrow-avro = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-avro" } +arrow-arith = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-arith" } diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 4c96293c..50d45438 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -19,10 +19,10 @@ async-trait = "0.1.88" chrono = "0.4.41" futures = "0.3.31" dashmap = "6.1.0" -prost = "0.14.0" +prost = "0.13" url = "2.5.4" arrow-flight = "57.1.0" -tonic = { version = "0.14.1", features = ["transport"] } +tonic = { version = "0.13", features = ["transport"] } axum = "0.7" object_store = { version = "0.12.4", features = ["aws"] } aws-config = "1" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 65c4339a..1302197f 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -12,7 +12,7 @@ clap = { version = "4", features = ["derive"] } env_logger = "0.11" dirs = "6" arrow-flight = "57.1.0" -tonic = { version = "0.14.1", features = ["transport"] } +tonic = { version = "0.13", features = ["transport"] } tower = "0.5.2" hyper-util = "0.1.16" tokio-stream = "0.1.17" From 5b7f92010c2ca7baa6bd80bba33d24c46a5f0b7b Mon Sep 17 00:00:00 2001 From: Gabriel Musat Mestre Date: Wed, 24 Dec 2025 11:59:07 +0100 Subject: [PATCH 22/27] Downgrade prost and tonic to 0.13 --- Cargo.lock | 74 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 63 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 11670c91..be1eab36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -327,9 +327,9 @@ dependencies = [ "base64", "bytes", "futures", - "prost", + "prost 0.14.1", "prost-types", - "tonic", + "tonic 0.14.2", ] [[package]] @@ -2054,13 +2054,13 @@ dependencies = [ "parquet", "pin-project", "pretty_assertions", - "prost", + "prost 0.13.5", "rand 0.8.5", "reqwest", "structopt", "tokio", "tokio-stream", - "tonic", + "tonic 0.13.1", "tower", "tpchgen", "tpchgen-arrow", @@ -2090,12 +2090,12 @@ dependencies = [ "object_store", "openssl", "parquet", - "prost", + "prost 0.13.5", "serde", "serde_json", "structopt", "tokio", - "tonic", + "tonic 0.13.1", "url", ] @@ -2114,7 +2114,7 @@ dependencies = [ "hyper-util", "tokio", "tokio-stream", - "tonic", + "tonic 0.13.1", "tower", "url", ] @@ -2451,7 +2451,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-proto-common", "object_store", - "prost", + "prost 0.14.1", ] [[package]] @@ -2461,7 +2461,7 @@ source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a00390 dependencies = [ "arrow", "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", - "prost", + "prost 0.14.1", ] [[package]] @@ -4597,7 +4597,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.13.5", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive 0.14.1", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.110", ] [[package]] @@ -4619,7 +4642,7 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ - "prost", + "prost 0.14.1", ] [[package]] @@ -5999,6 +6022,35 @@ dependencies = [ "winnow", ] +[[package]] +name = "tonic" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +dependencies = [ + "async-trait", + "axum 0.8.7", + "base64", + "bytes", + "h2 0.4.12", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "hyper 1.8.1", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost 0.13.5", + "socket2 0.5.10", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tonic" version = "0.13.1" From f50164d7ecabbf89d99b81b98a4a5cfb19dfaea0 Mon Sep 17 00:00:00 2001 From: Gabriel Musat Mestre Date: Wed, 24 Dec 2025 12:07:13 +0100 Subject: [PATCH 23/27] Patch dependencies with DD forks so that the project compiles --- Cargo.toml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index acf203b6..e4fc21e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,3 +102,24 @@ arrow-string = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd4 arrow-json = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-json" } arrow-avro = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-avro" } arrow-arith = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-arith" } + +[patch.crates-io] +datafusion = { git = "https://github.com/DataDog/datafusion", rev = "3ae66138c1d6551b29a0039059570c9927a27d52", package = "datafusion" } +datafusion-substrait = { git = "https://github.com/DataDog/datafusion", rev = "3ae66138c1d6551b29a0039059570c9927a27d52", package = "datafusion-substrait" } +datafusion-proto = { git = "https://github.com/DataDog/datafusion", rev = "3ae66138c1d6551b29a0039059570c9927a27d52", package = "datafusion-proto" } +datafusion-proto-common = { git = "https://github.com/DataDog/datafusion", rev = "3ae66138c1d6551b29a0039059570c9927a27d52", package = "datafusion-proto-common" } +arrow = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow" } +arrow-flight = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-flight" } +arrow-select = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-select" } +arrow-ord = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-ord" } +arrow-array = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-array" } +arrow-data = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-data" } +arrow-ipc = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-ipc" } +arrow-schema = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-schema" } +arrow-cast = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-cast" } +arrow-buffer = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-buffer" } +arrow-row = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-row" } +arrow-string = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-string" } +arrow-json = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-json" } +arrow-avro = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-avro" } +arrow-arith = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-arith" } From cad2c7601e4c624d7bba90163bf539b390ae35fe Mon Sep 17 00:00:00 2001 From: LiaCastaneda Date: Tue, 13 Jan 2026 16:36:53 +0100 Subject: [PATCH 24/27] Stop using arrow-rs patch branch --- Cargo.toml | 32 ++++++++++++++++---------------- benchmarks/Cargo.toml | 2 +- cli/Cargo.toml | 2 +- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e4fc21e2..11e1c3c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ arrow-flight = "57.1.0" arrow-select = "57.1.0" async-trait = "0.1.88" tokio = { version = "1.46.1", features = ["full"] } -tonic = { version = "0.13", features = ["transport"] } +tonic = { version = "0.14.1", features = ["transport"] } tower = "0.5.2" http = "1.3.1" itertools = "0.14.0" @@ -108,18 +108,18 @@ datafusion = { git = "https://github.com/DataDog/datafusion", rev = "3ae66138c1d datafusion-substrait = { git = "https://github.com/DataDog/datafusion", rev = "3ae66138c1d6551b29a0039059570c9927a27d52", package = "datafusion-substrait" } datafusion-proto = { git = "https://github.com/DataDog/datafusion", rev = "3ae66138c1d6551b29a0039059570c9927a27d52", package = "datafusion-proto" } datafusion-proto-common = { git = "https://github.com/DataDog/datafusion", rev = "3ae66138c1d6551b29a0039059570c9927a27d52", package = "datafusion-proto-common" } -arrow = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow" } -arrow-flight = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-flight" } -arrow-select = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-select" } -arrow-ord = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-ord" } -arrow-array = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-array" } -arrow-data = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-data" } -arrow-ipc = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-ipc" } -arrow-schema = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-schema" } -arrow-cast = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-cast" } -arrow-buffer = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-buffer" } -arrow-row = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-row" } -arrow-string = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-string" } -arrow-json = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-json" } -arrow-avro = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-avro" } -arrow-arith = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-arith" } +arrow = { git = "https://github.com/DataDog/arrow-rs", rev = "779e9bd2ee43d7d882782e6bf6a11ee0944af229", package = "arrow" } +arrow-flight = { git = "https://github.com/DataDog/arrow-rs", rev = "779e9bd2ee43d7d882782e6bf6a11ee0944af229", package = "arrow-flight" } +arrow-select = { git = "https://github.com/DataDog/arrow-rs", rev = "779e9bd2ee43d7d882782e6bf6a11ee0944af229", package = "arrow-select" } +arrow-ord = { git = "https://github.com/DataDog/arrow-rs", rev = "779e9bd2ee43d7d882782e6bf6a11ee0944af229", package = "arrow-ord" } +arrow-array = { git = "https://github.com/DataDog/arrow-rs", rev = "779e9bd2ee43d7d882782e6bf6a11ee0944af229", package = "arrow-array" } +arrow-data = { git = "https://github.com/DataDog/arrow-rs", rev = "779e9bd2ee43d7d882782e6bf6a11ee0944af229", package = "arrow-data" } +arrow-ipc = { git = "https://github.com/DataDog/arrow-rs", rev = "779e9bd2ee43d7d882782e6bf6a11ee0944af229", package = "arrow-ipc" } +arrow-schema = { git = "https://github.com/DataDog/arrow-rs", rev = "779e9bd2ee43d7d882782e6bf6a11ee0944af229", package = "arrow-schema" } +arrow-cast = { git = "https://github.com/DataDog/arrow-rs", rev = "779e9bd2ee43d7d882782e6bf6a11ee0944af229", package = "arrow-cast" } +arrow-buffer = { git = "https://github.com/DataDog/arrow-rs", rev = "779e9bd2ee43d7d882782e6bf6a11ee0944af229", package = "arrow-buffer" } +arrow-row = { git = "https://github.com/DataDog/arrow-rs", rev = "779e9bd2ee43d7d882782e6bf6a11ee0944af229", package = "arrow-row" } +arrow-string = { git = "https://github.com/DataDog/arrow-rs", rev = "779e9bd2ee43d7d882782e6bf6a11ee0944af229", package = "arrow-string" } +arrow-json = { git = "https://github.com/DataDog/arrow-rs", rev = "779e9bd2ee43d7d882782e6bf6a11ee0944af229", package = "arrow-json" } +arrow-avro = { git = "https://github.com/DataDog/arrow-rs", rev = "779e9bd2ee43d7d882782e6bf6a11ee0944af229", package = "arrow-avro" } +arrow-arith = { git = "https://github.com/DataDog/arrow-rs", rev = "779e9bd2ee43d7d882782e6bf6a11ee0944af229", package = "arrow-arith" } diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 50d45438..0ce65561 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -22,7 +22,7 @@ dashmap = "6.1.0" prost = "0.13" url = "2.5.4" arrow-flight = "57.1.0" -tonic = { version = "0.13", features = ["transport"] } +tonic = { version = "0.14.1", features = ["transport"] } axum = "0.7" object_store = { version = "0.12.4", features = ["aws"] } aws-config = "1" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 1302197f..65c4339a 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -12,7 +12,7 @@ clap = { version = "4", features = ["derive"] } env_logger = "0.11" dirs = "6" arrow-flight = "57.1.0" -tonic = { version = "0.13", features = ["transport"] } +tonic = { version = "0.14.1", features = ["transport"] } tower = "0.5.2" hyper-util = "0.1.16" tokio-stream = "0.1.17" From 885d8646fbda0a84eed979a27dc71eca8d9065d6 Mon Sep 17 00:00:00 2001 From: LiaCastaneda Date: Tue, 13 Jan 2026 18:44:04 +0100 Subject: [PATCH 25/27] Upgrade prost --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- benchmarks/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be1eab36..05371685 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2054,7 +2054,7 @@ dependencies = [ "parquet", "pin-project", "pretty_assertions", - "prost 0.13.5", + "prost 0.14.3", "rand 0.8.5", "reqwest", "structopt", @@ -2090,7 +2090,7 @@ dependencies = [ "object_store", "openssl", "parquet", - "prost 0.13.5", + "prost 0.14.3", "serde", "serde_json", "structopt", diff --git a/Cargo.toml b/Cargo.toml index 11e1c3c6..abfd8825 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ url = "2.5.4" uuid = "1.17.0" delegate = "0.13.4" dashmap = "6.1.0" -prost = "0.13" +prost = "0.14.0" rand = "0.8.5" object_store = "0.12.3" bytes = "1.10.1" diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 0ce65561..4c96293c 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -19,7 +19,7 @@ async-trait = "0.1.88" chrono = "0.4.41" futures = "0.3.31" dashmap = "6.1.0" -prost = "0.13" +prost = "0.14.0" url = "2.5.4" arrow-flight = "57.1.0" tonic = { version = "0.14.1", features = ["transport"] } From 36c852ca6fac7f5f381761f79654616ed909dfde Mon Sep 17 00:00:00 2001 From: LiaCastaneda Date: Tue, 13 Jan 2026 18:52:05 +0100 Subject: [PATCH 26/27] Fix DataFusion commit SHA --- Cargo.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index abfd8825..fe19493f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -104,10 +104,10 @@ arrow-avro = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433 arrow-arith = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-arith" } [patch.crates-io] -datafusion = { git = "https://github.com/DataDog/datafusion", rev = "3ae66138c1d6551b29a0039059570c9927a27d52", package = "datafusion" } -datafusion-substrait = { git = "https://github.com/DataDog/datafusion", rev = "3ae66138c1d6551b29a0039059570c9927a27d52", package = "datafusion-substrait" } -datafusion-proto = { git = "https://github.com/DataDog/datafusion", rev = "3ae66138c1d6551b29a0039059570c9927a27d52", package = "datafusion-proto" } -datafusion-proto-common = { git = "https://github.com/DataDog/datafusion", rev = "3ae66138c1d6551b29a0039059570c9927a27d52", package = "datafusion-proto-common" } +datafusion = { git = "https://github.com/DataDog/datafusion", rev = "507f6f5b0f4e0b1943f6f1b06615b4644ca438bc", package = "datafusion" } +datafusion-substrait = { git = "https://github.com/DataDog/datafusion", rev = "507f6f5b0f4e0b1943f6f1b06615b4644ca438bc", package = "datafusion-substrait" } +datafusion-proto = { git = "https://github.com/DataDog/datafusion", rev = "507f6f5b0f4e0b1943f6f1b06615b4644ca438bc", package = "datafusion-proto" } +datafusion-proto-common = { git = "https://github.com/DataDog/datafusion", rev = "507f6f5b0f4e0b1943f6f1b06615b4644ca438bc", package = "datafusion-proto-common" } arrow = { git = "https://github.com/DataDog/arrow-rs", rev = "779e9bd2ee43d7d882782e6bf6a11ee0944af229", package = "arrow" } arrow-flight = { git = "https://github.com/DataDog/arrow-rs", rev = "779e9bd2ee43d7d882782e6bf6a11ee0944af229", package = "arrow-flight" } arrow-select = { git = "https://github.com/DataDog/arrow-rs", rev = "779e9bd2ee43d7d882782e6bf6a11ee0944af229", package = "arrow-select" } From 7c618c33e146d8664c54a87b7f33dd99894f507a Mon Sep 17 00:00:00 2001 From: Joseph Koshakow Date: Wed, 14 Jan 2026 13:20:14 -0500 Subject: [PATCH 27/27] More rebases --- Cargo.lock | 254 ++++++++++++++++++++++------------------------------- Cargo.toml | 21 ----- 2 files changed, 107 insertions(+), 168 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 05371685..9b4b3417 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -208,7 +208,7 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" version = "57.1.0" -source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" +source = "git+https://github.com/DataDog/arrow-rs?rev=779e9bd2ee43d7d882782e6bf6a11ee0944af229#779e9bd2ee43d7d882782e6bf6a11ee0944af229" dependencies = [ "arrow-arith", "arrow-array", @@ -228,7 +228,7 @@ dependencies = [ [[package]] name = "arrow-arith" version = "57.1.0" -source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" +source = "git+https://github.com/DataDog/arrow-rs?rev=779e9bd2ee43d7d882782e6bf6a11ee0944af229#779e9bd2ee43d7d882782e6bf6a11ee0944af229" dependencies = [ "arrow-array", "arrow-buffer", @@ -241,7 +241,7 @@ dependencies = [ [[package]] name = "arrow-array" version = "57.1.0" -source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" +source = "git+https://github.com/DataDog/arrow-rs?rev=779e9bd2ee43d7d882782e6bf6a11ee0944af229#779e9bd2ee43d7d882782e6bf6a11ee0944af229" dependencies = [ "ahash", "arrow-buffer", @@ -259,7 +259,7 @@ dependencies = [ [[package]] name = "arrow-buffer" version = "57.1.0" -source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" +source = "git+https://github.com/DataDog/arrow-rs?rev=779e9bd2ee43d7d882782e6bf6a11ee0944af229#779e9bd2ee43d7d882782e6bf6a11ee0944af229" dependencies = [ "bytes", "half", @@ -270,7 +270,7 @@ dependencies = [ [[package]] name = "arrow-cast" version = "57.1.0" -source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" +source = "git+https://github.com/DataDog/arrow-rs?rev=779e9bd2ee43d7d882782e6bf6a11ee0944af229#779e9bd2ee43d7d882782e6bf6a11ee0944af229" dependencies = [ "arrow-array", "arrow-buffer", @@ -291,7 +291,7 @@ dependencies = [ [[package]] name = "arrow-csv" version = "57.1.0" -source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" +source = "git+https://github.com/DataDog/arrow-rs?rev=779e9bd2ee43d7d882782e6bf6a11ee0944af229#779e9bd2ee43d7d882782e6bf6a11ee0944af229" dependencies = [ "arrow-array", "arrow-cast", @@ -305,7 +305,7 @@ dependencies = [ [[package]] name = "arrow-data" version = "57.1.0" -source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" +source = "git+https://github.com/DataDog/arrow-rs?rev=779e9bd2ee43d7d882782e6bf6a11ee0944af229#779e9bd2ee43d7d882782e6bf6a11ee0944af229" dependencies = [ "arrow-buffer", "arrow-schema", @@ -317,7 +317,7 @@ dependencies = [ [[package]] name = "arrow-flight" version = "57.1.0" -source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" +source = "git+https://github.com/DataDog/arrow-rs?rev=779e9bd2ee43d7d882782e6bf6a11ee0944af229#779e9bd2ee43d7d882782e6bf6a11ee0944af229" dependencies = [ "arrow-array", "arrow-buffer", @@ -327,15 +327,16 @@ dependencies = [ "base64", "bytes", "futures", - "prost 0.14.1", + "prost", "prost-types", - "tonic 0.14.2", + "tonic", + "tonic-prost", ] [[package]] name = "arrow-ipc" version = "57.1.0" -source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" +source = "git+https://github.com/DataDog/arrow-rs?rev=779e9bd2ee43d7d882782e6bf6a11ee0944af229#779e9bd2ee43d7d882782e6bf6a11ee0944af229" dependencies = [ "arrow-array", "arrow-buffer", @@ -350,7 +351,7 @@ dependencies = [ [[package]] name = "arrow-json" version = "57.1.0" -source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" +source = "git+https://github.com/DataDog/arrow-rs?rev=779e9bd2ee43d7d882782e6bf6a11ee0944af229#779e9bd2ee43d7d882782e6bf6a11ee0944af229" dependencies = [ "arrow-array", "arrow-buffer", @@ -373,7 +374,7 @@ dependencies = [ [[package]] name = "arrow-ord" version = "57.1.0" -source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" +source = "git+https://github.com/DataDog/arrow-rs?rev=779e9bd2ee43d7d882782e6bf6a11ee0944af229#779e9bd2ee43d7d882782e6bf6a11ee0944af229" dependencies = [ "arrow-array", "arrow-buffer", @@ -385,7 +386,7 @@ dependencies = [ [[package]] name = "arrow-row" version = "57.1.0" -source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" +source = "git+https://github.com/DataDog/arrow-rs?rev=779e9bd2ee43d7d882782e6bf6a11ee0944af229#779e9bd2ee43d7d882782e6bf6a11ee0944af229" dependencies = [ "arrow-array", "arrow-buffer", @@ -397,7 +398,7 @@ dependencies = [ [[package]] name = "arrow-schema" version = "57.1.0" -source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" +source = "git+https://github.com/DataDog/arrow-rs?rev=779e9bd2ee43d7d882782e6bf6a11ee0944af229#779e9bd2ee43d7d882782e6bf6a11ee0944af229" dependencies = [ "serde_core", "serde_json", @@ -406,7 +407,7 @@ dependencies = [ [[package]] name = "arrow-select" version = "57.1.0" -source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" +source = "git+https://github.com/DataDog/arrow-rs?rev=779e9bd2ee43d7d882782e6bf6a11ee0944af229#779e9bd2ee43d7d882782e6bf6a11ee0944af229" dependencies = [ "ahash", "arrow-array", @@ -419,7 +420,7 @@ dependencies = [ [[package]] name = "arrow-string" version = "57.1.0" -source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" +source = "git+https://github.com/DataDog/arrow-rs?rev=779e9bd2ee43d7d882782e6bf6a11ee0944af229#779e9bd2ee43d7d882782e6bf6a11ee0944af229" dependencies = [ "arrow-array", "arrow-buffer", @@ -1699,7 +1700,7 @@ dependencies = [ [[package]] name = "datafusion" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", "arrow-schema", @@ -1709,7 +1710,7 @@ dependencies = [ "chrono", "datafusion-catalog", "datafusion-catalog-listing", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-common-runtime", "datafusion-datasource", "datafusion-datasource-arrow", @@ -1755,12 +1756,12 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-common-runtime", "datafusion-datasource", "datafusion-execution", @@ -1779,12 +1780,12 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", "async-trait", "datafusion-catalog", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-datasource", "datafusion-execution", "datafusion-expr", @@ -1850,7 +1851,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "ahash", "apache-avro", @@ -1875,7 +1876,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "futures", "log", @@ -1885,7 +1886,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", "async-compression", @@ -1893,7 +1894,7 @@ dependencies = [ "bytes", "bzip2", "chrono", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", @@ -1919,13 +1920,13 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", "arrow-ipc", "async-trait", "bytes", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-common-runtime", "datafusion-datasource", "datafusion-execution", @@ -1942,13 +1943,13 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "apache-avro", "arrow", "async-trait", "bytes", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-datasource", "datafusion-physical-expr-common", "datafusion-physical-plan", @@ -1961,12 +1962,12 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-common-runtime", "datafusion-datasource", "datafusion-execution", @@ -1983,12 +1984,12 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-common-runtime", "datafusion-datasource", "datafusion-execution", @@ -2004,12 +2005,12 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-common-runtime", "datafusion-datasource", "datafusion-execution", @@ -2054,13 +2055,13 @@ dependencies = [ "parquet", "pin-project", "pretty_assertions", - "prost 0.14.3", + "prost", "rand 0.8.5", "reqwest", "structopt", "tokio", "tokio-stream", - "tonic 0.13.1", + "tonic", "tower", "tpchgen", "tpchgen-arrow", @@ -2090,12 +2091,12 @@ dependencies = [ "object_store", "openssl", "parquet", - "prost 0.14.3", + "prost", "serde", "serde_json", "structopt", "tokio", - "tonic 0.13.1", + "tonic", "url", ] @@ -2114,7 +2115,7 @@ dependencies = [ "hyper-util", "tokio", "tokio-stream", - "tonic 0.13.1", + "tonic", "tower", "url", ] @@ -2122,17 +2123,17 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" [[package]] name = "datafusion-execution" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-expr", "futures", "log", @@ -2147,12 +2148,12 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", "async-trait", "chrono", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-doc", "datafusion-expr-common", "datafusion-functions-aggregate-common", @@ -2169,10 +2170,10 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "indexmap", "itertools 0.14.0", "paste", @@ -2181,7 +2182,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", "arrow-buffer", @@ -2189,7 +2190,7 @@ dependencies = [ "blake2", "blake3", "chrono", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-doc", "datafusion-execution", "datafusion-expr", @@ -2210,11 +2211,11 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "ahash", "arrow", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-doc", "datafusion-execution", "datafusion-expr", @@ -2230,11 +2231,11 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "ahash", "arrow", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-expr-common", "datafusion-physical-expr-common", ] @@ -2242,11 +2243,11 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", "arrow-ord", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-doc", "datafusion-execution", "datafusion-expr", @@ -2264,12 +2265,12 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", "async-trait", "datafusion-catalog", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-expr", "datafusion-physical-plan", "parking_lot", @@ -2279,10 +2280,10 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-doc", "datafusion-expr", "datafusion-functions-window-common", @@ -2296,16 +2297,16 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-physical-expr-common", ] [[package]] name = "datafusion-macros" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "datafusion-doc", "quote", @@ -2315,11 +2316,11 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", "chrono", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-expr", "datafusion-expr-common", "datafusion-physical-expr", @@ -2334,11 +2335,11 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "ahash", "arrow", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-expr", "datafusion-expr-common", "datafusion-functions-aggregate-common", @@ -2356,10 +2357,10 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-expr", "datafusion-functions", "datafusion-physical-expr", @@ -2370,11 +2371,11 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "ahash", "arrow", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-expr-common", "hashbrown 0.14.5", "itertools 0.14.0", @@ -2383,10 +2384,10 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-execution", "datafusion-expr", "datafusion-expr-common", @@ -2401,7 +2402,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "ahash", "arrow", @@ -2409,7 +2410,7 @@ dependencies = [ "arrow-schema", "async-trait", "chrono", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", @@ -2431,13 +2432,13 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", "chrono", "datafusion-catalog", "datafusion-catalog-listing", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-datasource", "datafusion-datasource-arrow", "datafusion-datasource-csv", @@ -2451,26 +2452,26 @@ dependencies = [ "datafusion-physical-plan", "datafusion-proto-common", "object_store", - "prost 0.14.1", + "prost", ] [[package]] name = "datafusion-proto-common" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", - "prost 0.14.1", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", + "prost", ] [[package]] name = "datafusion-pruning" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-datasource", "datafusion-expr-common", "datafusion-physical-expr", @@ -2483,10 +2484,10 @@ dependencies = [ [[package]] name = "datafusion-session" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "async-trait", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-execution", "datafusion-expr", "datafusion-physical-plan", @@ -2496,12 +2497,12 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" dependencies = [ "arrow", "bigdecimal", "chrono", - "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52)", + "datafusion-common 51.0.0 (git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc)", "datafusion-expr", "indexmap", "log", @@ -4592,42 +4593,19 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" -dependencies = [ - "bytes", - "prost-derive 0.13.5", -] - -[[package]] -name = "prost" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" dependencies = [ "bytes", - "prost-derive 0.14.1", + "prost-derive", ] [[package]] name = "prost-derive" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn 2.0.110", -] - -[[package]] -name = "prost-derive" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", "itertools 0.14.0", @@ -4638,11 +4616,11 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" dependencies = [ - "prost 0.14.1", + "prost", ] [[package]] @@ -6024,9 +6002,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.13.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" dependencies = [ "async-trait", "axum 0.8.7", @@ -6041,8 +6019,8 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost 0.13.5", - "socket2 0.5.10", + "socket2 0.6.1", + "sync_wrapper", "tokio", "tokio-stream", "tower", @@ -6052,32 +6030,14 @@ dependencies = [ ] [[package]] -name = "tonic" -version = "0.13.1" +name = "tonic-prost" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" dependencies = [ - "async-trait", - "axum 0.8.7", - "base64", "bytes", - "h2 0.4.12", - "http 1.3.1", - "http-body 1.0.1", - "http-body-util", - "hyper 1.8.1", - "hyper-timeout", - "hyper-util", - "percent-encoding", - "pin-project", "prost", - "socket2 0.5.10", - "tokio", - "tokio-stream", - "tower", - "tower-layer", - "tower-service", - "tracing", + "tonic", ] [[package]] @@ -7033,9 +6993,9 @@ dependencies = [ [[patch.unused]] name = "arrow-avro" version = "57.1.0" -source = "git+https://github.com/DataDog/arrow-rs?rev=dd2c5a10fd433374c896d62e9e4fee92b4717a7e#dd2c5a10fd433374c896d62e9e4fee92b4717a7e" +source = "git+https://github.com/DataDog/arrow-rs?rev=779e9bd2ee43d7d882782e6bf6a11ee0944af229#779e9bd2ee43d7d882782e6bf6a11ee0944af229" [[patch.unused]] name = "datafusion-substrait" version = "51.0.0" -source = "git+https://github.com/DataDog/datafusion?rev=3ae66138c1d6551b29a0039059570c9927a27d52#3ae66138c1d6551b29a0039059570c9927a27d52" +source = "git+https://github.com/DataDog/datafusion?rev=507f6f5b0f4e0b1943f6f1b06615b4644ca438bc#507f6f5b0f4e0b1943f6f1b06615b4644ca438bc" diff --git a/Cargo.toml b/Cargo.toml index fe19493f..054b41b2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,27 +82,6 @@ pretty_assertions = "1.4" reqwest = "0.12" zip = "4.0" -[patch.crates-io] -datafusion = { git = "https://github.com/DataDog/datafusion", rev = "3ae66138c1d6551b29a0039059570c9927a27d52", package = "datafusion" } -datafusion-substrait = { git = "https://github.com/DataDog/datafusion", rev = "3ae66138c1d6551b29a0039059570c9927a27d52", package = "datafusion-substrait" } -datafusion-proto = { git = "https://github.com/DataDog/datafusion", rev = "3ae66138c1d6551b29a0039059570c9927a27d52", package = "datafusion-proto" } -datafusion-proto-common = { git = "https://github.com/DataDog/datafusion", rev = "3ae66138c1d6551b29a0039059570c9927a27d52", package = "datafusion-proto-common" } -arrow = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow" } -arrow-flight = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-flight" } -arrow-select = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-select" } -arrow-ord = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-ord" } -arrow-array = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-array" } -arrow-data = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-data" } -arrow-ipc = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-ipc" } -arrow-schema = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-schema" } -arrow-cast = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-cast" } -arrow-buffer = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-buffer" } -arrow-row = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-row" } -arrow-string = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-string" } -arrow-json = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-json" } -arrow-avro = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-avro" } -arrow-arith = { git = "https://github.com/DataDog/arrow-rs", rev = "dd2c5a10fd433374c896d62e9e4fee92b4717a7e", package = "arrow-arith" } - [patch.crates-io] datafusion = { git = "https://github.com/DataDog/datafusion", rev = "507f6f5b0f4e0b1943f6f1b06615b4644ca438bc", package = "datafusion" } datafusion-substrait = { git = "https://github.com/DataDog/datafusion", rev = "507f6f5b0f4e0b1943f6f1b06615b4644ca438bc", package = "datafusion-substrait" }